{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:A576KJN25IHFY4HTDYGTY2FTP5","short_pith_number":"pith:A576KJN2","schema_version":"1.0","canonical_sha256":"077fe525baea0e5c70f31e0d3c68b37f7a6e2b50d4303220d5ce5b43f3ad67e8","source":{"kind":"arxiv","id":"2010.05137","version":2},"attestation_state":"computed","paper":{"title":"An Open Review of OpenReview: A Critical Analysis of the Machine Learning Conference Review Process","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.LG","authors_text":"Alex Valtchanov, David Tran, Eric Slud, Keshav Ganapathy, Micah Goldblum, Raymond Feng, Tom Goldstein","submitted_at":"2020-10-11T02:06:04Z","abstract_excerpt":"Mainstream machine learning conferences have seen a dramatic increase in the number of participants, along with a growing range of perspectives, in recent years. Members of the machine learning community are likely to overhear allegations ranging from randomness of acceptance decisions to institutional bias. In this work, we critically analyze the review process through a comprehensive study of papers submitted to ICLR between 2017 and 2020. We quantify reproducibility/randomness in review scores and acceptance decisions, and examine whether scores correlate with paper impact. Our findings sug"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2010.05137","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-11T02:06:04Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"43b3dffa9e397310cf36c72db654ecce77845701da21a5445e99d478ac01b30f","abstract_canon_sha256":"dce40271aef233fefa4173bf4e7e226d75d698d1be09bbc6665c3d53939f398f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:46:27.158665Z","signature_b64":"M2rkZbgKKjNJg8tuzl9QEhcB4BLSbDg4pago6WkHbxknsjKoM7ghIO2f1knek+mmlYa/WLf2B3nQAzAnZgqNBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"077fe525baea0e5c70f31e0d3c68b37f7a6e2b50d4303220d5ce5b43f3ad67e8","last_reissued_at":"2026-07-05T01:46:27.158178Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:46:27.158178Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"An Open Review of OpenReview: A Critical Analysis of the Machine Learning Conference Review Process","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.LG","authors_text":"Alex Valtchanov, David Tran, Eric Slud, Keshav Ganapathy, Micah Goldblum, Raymond Feng, Tom Goldstein","submitted_at":"2020-10-11T02:06:04Z","abstract_excerpt":"Mainstream machine learning conferences have seen a dramatic increase in the number of participants, along with a growing range of perspectives, in recent years. Members of the machine learning community are likely to overhear allegations ranging from randomness of acceptance decisions to institutional bias. In this work, we critically analyze the review process through a comprehensive study of papers submitted to ICLR between 2017 and 2020. We quantify reproducibility/randomness in review scores and acceptance decisions, and examine whether scores correlate with paper impact. Our findings sug"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.05137","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.05137/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2010.05137","created_at":"2026-07-05T01:46:27.158235+00:00"},{"alias_kind":"arxiv_version","alias_value":"2010.05137v2","created_at":"2026-07-05T01:46:27.158235+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.05137","created_at":"2026-07-05T01:46:27.158235+00:00"},{"alias_kind":"pith_short_12","alias_value":"A576KJN25IHF","created_at":"2026-07-05T01:46:27.158235+00:00"},{"alias_kind":"pith_short_16","alias_value":"A576KJN25IHFY4HT","created_at":"2026-07-05T01:46:27.158235+00:00"},{"alias_kind":"pith_short_8","alias_value":"A576KJN2","created_at":"2026-07-05T01:46:27.158235+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.07119","citing_title":"Are Non-English Papers Reviewed Fairly? Language-of-Study Bias in NLP Peer Reviews","ref_index":2,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5","json":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5.json","graph_json":"https://pith.science/api/pith-number/A576KJN25IHFY4HTDYGTY2FTP5/graph.json","events_json":"https://pith.science/api/pith-number/A576KJN25IHFY4HTDYGTY2FTP5/events.json","paper":"https://pith.science/paper/A576KJN2"},"agent_actions":{"view_html":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5","download_json":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5.json","view_paper":"https://pith.science/paper/A576KJN2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2010.05137&json=true","fetch_graph":"https://pith.science/api/pith-number/A576KJN25IHFY4HTDYGTY2FTP5/graph.json","fetch_events":"https://pith.science/api/pith-number/A576KJN25IHFY4HTDYGTY2FTP5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5/action/storage_attestation","attest_author":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5/action/author_attestation","sign_citation":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5/action/citation_signature","submit_replication":"https://pith.science/pith/A576KJN25IHFY4HTDYGTY2FTP5/action/replication_record"}},"created_at":"2026-07-05T01:46:27.158235+00:00","updated_at":"2026-07-05T01:46:27.158235+00:00"}