{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:OEWVM22MHSVT5OO667F7HHTC7Y","short_pith_number":"pith:OEWVM22M","canonical_record":{"source":{"id":"2306.03262","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T21:26:12Z","cross_cats_sorted":["cs.DL"],"title_canon_sha256":"d21182781c5dca11b5ddc82a40d450814297f5d03439a71acb8e7f7e9cbd6b4e","abstract_canon_sha256":"60fc30e86f46e9a6c17b8016c60b2b06c6f001ad6d1f9e31178dbb859362baf2"},"schema_version":"1.0"},"canonical_sha256":"712d566b4c3cab3eb9def7cbf39e62fe0d6c6f693eb6c68504309eefcd20cbbf","source":{"kind":"arxiv","id":"2306.03262","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.03262","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"arxiv_version","alias_value":"2306.03262v1","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.03262","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_12","alias_value":"OEWVM22MHSVT","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_16","alias_value":"OEWVM22MHSVT5OO6","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_8","alias_value":"OEWVM22M","created_at":"2026-07-05T06:17:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:OEWVM22MHSVT5OO667F7HHTC7Y","target":"record","payload":{"canonical_record":{"source":{"id":"2306.03262","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T21:26:12Z","cross_cats_sorted":["cs.DL"],"title_canon_sha256":"d21182781c5dca11b5ddc82a40d450814297f5d03439a71acb8e7f7e9cbd6b4e","abstract_canon_sha256":"60fc30e86f46e9a6c17b8016c60b2b06c6f001ad6d1f9e31178dbb859362baf2"},"schema_version":"1.0"},"canonical_sha256":"712d566b4c3cab3eb9def7cbf39e62fe0d6c6f693eb6c68504309eefcd20cbbf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:17:55.671476Z","signature_b64":"kIEgDwjIUVAj2wTKlne/cfO3PTVX+By6bKRaz+IegQph2A6+dc36JPOUBx/7Ti0SFgcIhEcO7fv6aQnwW4G8Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"712d566b4c3cab3eb9def7cbf39e62fe0d6c6f693eb6c68504309eefcd20cbbf","last_reissued_at":"2026-07-05T06:17:55.670990Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:17:55.670990Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2306.03262","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:17:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zZfQ5MWSKcodmxYPMFGxCO1EUCRSCTKQ+mzwPyyfDHjHw+HhIMlDuHX36aT7H6umoA4XAh2rkoVzPIzp9sgbCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T23:38:02.265134Z"},"content_sha256":"3dde1629958cf08c1ea47c565b8afedd9203d3922fe3d95c234ef2539019aaa5","schema_version":"1.0","event_id":"sha256:3dde1629958cf08c1ea47c565b8afedd9203d3922fe3d95c234ef2539019aaa5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:OEWVM22MHSVT5OO667F7HHTC7Y","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Has the Machine Learning Review Process Become More Arbitrary as the Field Has Grown? The NeurIPS 2021 Consistency Experiment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.DL"],"primary_cat":"cs.LG","authors_text":"Alina Beygelzimer, Jennifer Wortman Vaughan, Percy Liang, Yann N. Dauphin","submitted_at":"2023-06-05T21:26:12Z","abstract_excerpt":"We present the NeurIPS 2021 consistency experiment, a larger-scale variant of the 2014 NeurIPS experiment in which 10% of conference submissions were reviewed by two independent committees to quantify the randomness in the review process. We observe that the two committees disagree on their accept/reject recommendations for 23% of the papers and that, consistent with the results from 2014, approximately half of the list of accepted papers would change if the review process were randomly rerun. Our analysis suggests that making the conference more selective would increase the arbitrariness of t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.03262","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.03262/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:17:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iQgCeNic404MDowCF4hGmOBbIiXvKCMA7qeEYj3O+KgmK0TYG/+FD9kigllewcVlRY1ev1R8qDbsoiPNIHzJDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T23:38:02.265768Z"},"content_sha256":"7eccb9dbfc42c1a210b1d621facbf8108d931fa6dd3012261500cff650f2acf0","schema_version":"1.0","event_id":"sha256:7eccb9dbfc42c1a210b1d621facbf8108d931fa6dd3012261500cff650f2acf0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OEWVM22MHSVT5OO667F7HHTC7Y/bundle.json","state_url":"https://pith.science/pith/OEWVM22MHSVT5OO667F7HHTC7Y/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OEWVM22MHSVT5OO667F7HHTC7Y/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-11T23:38:02Z","links":{"resolver":"https://pith.science/pith/OEWVM22MHSVT5OO667F7HHTC7Y","bundle":"https://pith.science/pith/OEWVM22MHSVT5OO667F7HHTC7Y/bundle.json","state":"https://pith.science/pith/OEWVM22MHSVT5OO667F7HHTC7Y/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OEWVM22MHSVT5OO667F7HHTC7Y/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:OEWVM22MHSVT5OO667F7HHTC7Y","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"60fc30e86f46e9a6c17b8016c60b2b06c6f001ad6d1f9e31178dbb859362baf2","cross_cats_sorted":["cs.DL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T21:26:12Z","title_canon_sha256":"d21182781c5dca11b5ddc82a40d450814297f5d03439a71acb8e7f7e9cbd6b4e"},"schema_version":"1.0","source":{"id":"2306.03262","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.03262","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"arxiv_version","alias_value":"2306.03262v1","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.03262","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_12","alias_value":"OEWVM22MHSVT","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_16","alias_value":"OEWVM22MHSVT5OO6","created_at":"2026-07-05T06:17:55Z"},{"alias_kind":"pith_short_8","alias_value":"OEWVM22M","created_at":"2026-07-05T06:17:55Z"}],"graph_snapshots":[{"event_id":"sha256:7eccb9dbfc42c1a210b1d621facbf8108d931fa6dd3012261500cff650f2acf0","target":"graph","created_at":"2026-07-05T06:17:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2306.03262/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present the NeurIPS 2021 consistency experiment, a larger-scale variant of the 2014 NeurIPS experiment in which 10% of conference submissions were reviewed by two independent committees to quantify the randomness in the review process. We observe that the two committees disagree on their accept/reject recommendations for 23% of the papers and that, consistent with the results from 2014, approximately half of the list of accepted papers would change if the review process were randomly rerun. Our analysis suggests that making the conference more selective would increase the arbitrariness of t","authors_text":"Alina Beygelzimer, Jennifer Wortman Vaughan, Percy Liang, Yann N. Dauphin","cross_cats":["cs.DL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T21:26:12Z","title":"Has the Machine Learning Review Process Become More Arbitrary as the Field Has Grown? The NeurIPS 2021 Consistency Experiment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.03262","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3dde1629958cf08c1ea47c565b8afedd9203d3922fe3d95c234ef2539019aaa5","target":"record","created_at":"2026-07-05T06:17:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"60fc30e86f46e9a6c17b8016c60b2b06c6f001ad6d1f9e31178dbb859362baf2","cross_cats_sorted":["cs.DL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T21:26:12Z","title_canon_sha256":"d21182781c5dca11b5ddc82a40d450814297f5d03439a71acb8e7f7e9cbd6b4e"},"schema_version":"1.0","source":{"id":"2306.03262","kind":"arxiv","version":1}},"canonical_sha256":"712d566b4c3cab3eb9def7cbf39e62fe0d6c6f693eb6c68504309eefcd20cbbf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"712d566b4c3cab3eb9def7cbf39e62fe0d6c6f693eb6c68504309eefcd20cbbf","first_computed_at":"2026-07-05T06:17:55.670990Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:17:55.670990Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kIEgDwjIUVAj2wTKlne/cfO3PTVX+By6bKRaz+IegQph2A6+dc36JPOUBx/7Ti0SFgcIhEcO7fv6aQnwW4G8Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T06:17:55.671476Z","signed_message":"canonical_sha256_bytes"},"source_id":"2306.03262","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3dde1629958cf08c1ea47c565b8afedd9203d3922fe3d95c234ef2539019aaa5","sha256:7eccb9dbfc42c1a210b1d621facbf8108d931fa6dd3012261500cff650f2acf0"],"state_sha256":"f9b1186ba1b439bbef66f07ce7a0b3ed65cc65e463b2698418b4000656a76840"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lwjfQ0zrdQisqqulgbeanDrn0uHsMqEhzxvFblKohUoBZ6W3yGHgvz3x0Hqfi2Aid+46P+ZM/lGxRqgf+UlwDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-11T23:38:02.269642Z","bundle_sha256":"e97c0218d67b7c177e13d568cd3fea64b1020b169174d837409f7a4573749967"}}