{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:LJATEM5CHARQSNINXSYFCLRXZH","short_pith_number":"pith:LJATEM5C","schema_version":"1.0","canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","source":{"kind":"arxiv","id":"2107.07002","version":1},"attestation_state":"computed","paper":{"title":"The Benchmark Lottery","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.CV","cs.IR"],"primary_cat":"cs.LG","authors_text":"Alexey A. Gritsenko, Donald Metzler, Fernando Diaz, Mostafa Dehghani, Neil Houlsby, Oriol Vinyals, Yi Tay, Zhe Zhao","submitted_at":"2021-07-14T21:08:30Z","abstract_excerpt":"The world of empirical machine learning (ML) strongly relies on benchmarks in order to determine the relative effectiveness of different algorithms and methods. This paper proposes the notion of \"a benchmark lottery\" that describes the overall fragility of the ML benchmarking process. The benchmark lottery postulates that many factors, other than fundamental algorithmic superiority, may lead to a method being perceived as superior. On multiple benchmark setups that are prevalent in the ML community, we show that the relative performance of algorithms may be altered significantly simply by choo"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2107.07002","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","cs.IR"],"title_canon_sha256":"61ee46f247c9341e1690cbc162b91b10023053934632014dc578e840ec83ae64","abstract_canon_sha256":"57854b22c913474b70b8159ce6983723b209e92e1a5c8a2da4d29fdd6694f101"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:58:27.172418Z","signature_b64":"RPH54GvHURQfg56QJailKcw3h0veP4JpLCyeTo0QGKh1mLwZDA4a7CYNBsDCEistHBnbGkFTZto1BFnGsrGbAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","last_reissued_at":"2026-07-05T02:58:27.171990Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:58:27.171990Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Benchmark Lottery","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.CV","cs.IR"],"primary_cat":"cs.LG","authors_text":"Alexey A. Gritsenko, Donald Metzler, Fernando Diaz, Mostafa Dehghani, Neil Houlsby, Oriol Vinyals, Yi Tay, Zhe Zhao","submitted_at":"2021-07-14T21:08:30Z","abstract_excerpt":"The world of empirical machine learning (ML) strongly relies on benchmarks in order to determine the relative effectiveness of different algorithms and methods. This paper proposes the notion of \"a benchmark lottery\" that describes the overall fragility of the ML benchmarking process. The benchmark lottery postulates that many factors, other than fundamental algorithmic superiority, may lead to a method being perceived as superior. On multiple benchmark setups that are prevalent in the ML community, we show that the relative performance of algorithms may be altered significantly simply by choo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.07002","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.07002/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2107.07002","created_at":"2026-07-05T02:58:27.172046+00:00"},{"alias_kind":"arxiv_version","alias_value":"2107.07002v1","created_at":"2026-07-05T02:58:27.172046+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.07002","created_at":"2026-07-05T02:58:27.172046+00:00"},{"alias_kind":"pith_short_12","alias_value":"LJATEM5CHARQ","created_at":"2026-07-05T02:58:27.172046+00:00"},{"alias_kind":"pith_short_16","alias_value":"LJATEM5CHARQSNIN","created_at":"2026-07-05T02:58:27.172046+00:00"},{"alias_kind":"pith_short_8","alias_value":"LJATEM5C","created_at":"2026-07-05T02:58:27.172046+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":22,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.19704","citing_title":"Beyond Static Leaderboards: Predictive Validity for the Evaluation of LLM Agents","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.17799","citing_title":"Position: Coding Benchmarks Are Misaligned with Agentic Software Engineering","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05029","citing_title":"Validity Threats for Foundation Model Research","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24213","citing_title":"Towards Evaluation Engineering: An Empirical Study of ML Evaluation Harnesses in the Wild","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29159","citing_title":"Pooled Leaderboards Hide System-Specific Winners: A Reporting-Protocol Audit of Offline Root-Cause Analysis Benchmarks","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27997","citing_title":"Benchmarking on Tasks That Matter: Dataset Selection for Preserving Model Rankings","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00135","citing_title":"On Effectiveness and Efficiency of Agentic Tool-calling and RL Training","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29335","citing_title":"Rethinking FID Through the Geometry of the Reference Dataset","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30916","citing_title":"Welfare, Improvability, and Variance: A Principal-Agent Approach to Optimal Benchmark Item Aggregation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30836","citing_title":"Cross-Layer Subspace Coupling for LLM Compression: A Unifying Framework and Its Empirical Limits","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.02258","citing_title":"Matter to Mechanism: A Benchmark for AI Co-Scientists in Materials and Battery Research","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23628","citing_title":"How Hard is it to Rig a Benchmark? A Social Choice Analysis of Leaderboard Robustness","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18229","citing_title":"Are Sparse Autoencoder Benchmarks Reliable?","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13988","citing_title":"Neural Fields for NV-Center Inverse Sensing","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12678","citing_title":"No One Knows the State of the Art in Geospatial Foundation Models","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12673","citing_title":"Do Androids Dream of Breaking the Game? Systematically Auditing AI Agent Benchmarks with BenchJack","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10834","citing_title":"From Controlled to the Wild: Evaluation of Pentesting Agents for the Real-World","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10546","citing_title":"Higher Resolution, Better Generalization: Unlocking Visual Scaling in Deep Reinforcement Learning","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23102","citing_title":"Unstable Rankings in Bayesian Deep Learning Evaluation","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2108.07258","citing_title":"On the Opportunities and Risks of Foundation Models","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.21930","citing_title":"Temporal Taskification in Streaming Continual Learning: A Source of Evaluation Instability","ref_index":47,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02442","citing_title":"Measuring AI Reasoning: A Guide for Researchers","ref_index":149,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH","json":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH.json","graph_json":"https://pith.science/api/pith-number/LJATEM5CHARQSNINXSYFCLRXZH/graph.json","events_json":"https://pith.science/api/pith-number/LJATEM5CHARQSNINXSYFCLRXZH/events.json","paper":"https://pith.science/paper/LJATEM5C"},"agent_actions":{"view_html":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH","download_json":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH.json","view_paper":"https://pith.science/paper/LJATEM5C","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2107.07002&json=true","fetch_graph":"https://pith.science/api/pith-number/LJATEM5CHARQSNINXSYFCLRXZH/graph.json","fetch_events":"https://pith.science/api/pith-number/LJATEM5CHARQSNINXSYFCLRXZH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/action/storage_attestation","attest_author":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/action/author_attestation","sign_citation":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/action/citation_signature","submit_replication":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/action/replication_record"}},"created_at":"2026-07-05T02:58:27.172046+00:00","updated_at":"2026-07-05T02:58:27.172046+00:00"}