{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:LJATEM5CHARQSNINXSYFCLRXZH","short_pith_number":"pith:LJATEM5C","canonical_record":{"source":{"id":"2107.07002","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","cs.IR"],"title_canon_sha256":"61ee46f247c9341e1690cbc162b91b10023053934632014dc578e840ec83ae64","abstract_canon_sha256":"57854b22c913474b70b8159ce6983723b209e92e1a5c8a2da4d29fdd6694f101"},"schema_version":"1.0"},"canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","source":{"kind":"arxiv","id":"2107.07002","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.07002","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"arxiv_version","alias_value":"2107.07002v1","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.07002","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_12","alias_value":"LJATEM5CHARQ","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_16","alias_value":"LJATEM5CHARQSNIN","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_8","alias_value":"LJATEM5C","created_at":"2026-07-05T02:58:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:LJATEM5CHARQSNINXSYFCLRXZH","target":"record","payload":{"canonical_record":{"source":{"id":"2107.07002","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","cs.IR"],"title_canon_sha256":"61ee46f247c9341e1690cbc162b91b10023053934632014dc578e840ec83ae64","abstract_canon_sha256":"57854b22c913474b70b8159ce6983723b209e92e1a5c8a2da4d29fdd6694f101"},"schema_version":"1.0"},"canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:58:27.172418Z","signature_b64":"RPH54GvHURQfg56QJailKcw3h0veP4JpLCyeTo0QGKh1mLwZDA4a7CYNBsDCEistHBnbGkFTZto1BFnGsrGbAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","last_reissued_at":"2026-07-05T02:58:27.171990Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:58:27.171990Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.07002","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OzM1GDVYY1GNv3io6bAOyvb61pJkLmYcfJUE0P9wSAHvGipy52hHJKx2utkvu5e99bsWr13KC4uaNj/aZ5kMBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T14:51:14.142948Z"},"content_sha256":"b8afe06fe4dc630a182f29e572cd722b593a74eefd4914c460311dfdfb6fd3ff","schema_version":"1.0","event_id":"sha256:b8afe06fe4dc630a182f29e572cd722b593a74eefd4914c460311dfdfb6fd3ff"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:LJATEM5CHARQSNINXSYFCLRXZH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Benchmark Lottery","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.CV","cs.IR"],"primary_cat":"cs.LG","authors_text":"Alexey A. Gritsenko, Donald Metzler, Fernando Diaz, Mostafa Dehghani, Neil Houlsby, Oriol Vinyals, Yi Tay, Zhe Zhao","submitted_at":"2021-07-14T21:08:30Z","abstract_excerpt":"The world of empirical machine learning (ML) strongly relies on benchmarks in order to determine the relative effectiveness of different algorithms and methods. This paper proposes the notion of \"a benchmark lottery\" that describes the overall fragility of the ML benchmarking process. The benchmark lottery postulates that many factors, other than fundamental algorithmic superiority, may lead to a method being perceived as superior. On multiple benchmark setups that are prevalent in the ML community, we show that the relative performance of algorithms may be altered significantly simply by choo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.07002","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.07002/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cu2wyPrQVhL3xoHuskdXi+0b8VB3JHV9q1EO4vnqg3FSrf3FMxH5hjDwqy1IGoX47flTfznTtGxtyYjXTWq1Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T14:51:14.143339Z"},"content_sha256":"43c718d936e880a2af6cc757c4b2f8323e78c5babe3794de56f23f7fa80f6120","schema_version":"1.0","event_id":"sha256:43c718d936e880a2af6cc757c4b2f8323e78c5babe3794de56f23f7fa80f6120"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/bundle.json","state_url":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LJATEM5CHARQSNINXSYFCLRXZH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-10T14:51:14Z","links":{"resolver":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH","bundle":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/bundle.json","state":"https://pith.science/pith/LJATEM5CHARQSNINXSYFCLRXZH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LJATEM5CHARQSNINXSYFCLRXZH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:LJATEM5CHARQSNINXSYFCLRXZH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"57854b22c913474b70b8159ce6983723b209e92e1a5c8a2da4d29fdd6694f101","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","cs.IR"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","title_canon_sha256":"61ee46f247c9341e1690cbc162b91b10023053934632014dc578e840ec83ae64"},"schema_version":"1.0","source":{"id":"2107.07002","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.07002","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"arxiv_version","alias_value":"2107.07002v1","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.07002","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_12","alias_value":"LJATEM5CHARQ","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_16","alias_value":"LJATEM5CHARQSNIN","created_at":"2026-07-05T02:58:27Z"},{"alias_kind":"pith_short_8","alias_value":"LJATEM5C","created_at":"2026-07-05T02:58:27Z"}],"graph_snapshots":[{"event_id":"sha256:43c718d936e880a2af6cc757c4b2f8323e78c5babe3794de56f23f7fa80f6120","target":"graph","created_at":"2026-07-05T02:58:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.07002/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The world of empirical machine learning (ML) strongly relies on benchmarks in order to determine the relative effectiveness of different algorithms and methods. This paper proposes the notion of \"a benchmark lottery\" that describes the overall fragility of the ML benchmarking process. The benchmark lottery postulates that many factors, other than fundamental algorithmic superiority, may lead to a method being perceived as superior. On multiple benchmark setups that are prevalent in the ML community, we show that the relative performance of algorithms may be altered significantly simply by choo","authors_text":"Alexey A. Gritsenko, Donald Metzler, Fernando Diaz, Mostafa Dehghani, Neil Houlsby, Oriol Vinyals, Yi Tay, Zhe Zhao","cross_cats":["cs.AI","cs.CL","cs.CV","cs.IR"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","title":"The Benchmark Lottery"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.07002","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b8afe06fe4dc630a182f29e572cd722b593a74eefd4914c460311dfdfb6fd3ff","target":"record","created_at":"2026-07-05T02:58:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"57854b22c913474b70b8159ce6983723b209e92e1a5c8a2da4d29fdd6694f101","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","cs.IR"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-14T21:08:30Z","title_canon_sha256":"61ee46f247c9341e1690cbc162b91b10023053934632014dc578e840ec83ae64"},"schema_version":"1.0","source":{"id":"2107.07002","kind":"arxiv","version":1}},"canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5a413233a2382309350dbcb0512e37c9c6dffc66151e017159b11151bc0eef15","first_computed_at":"2026-07-05T02:58:27.171990Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:58:27.171990Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"RPH54GvHURQfg56QJailKcw3h0veP4JpLCyeTo0QGKh1mLwZDA4a7CYNBsDCEistHBnbGkFTZto1BFnGsrGbAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T02:58:27.172418Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.07002","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b8afe06fe4dc630a182f29e572cd722b593a74eefd4914c460311dfdfb6fd3ff","sha256:43c718d936e880a2af6cc757c4b2f8323e78c5babe3794de56f23f7fa80f6120"],"state_sha256":"c296b25fbec236903cae47dca082529b694a82244357196538003f8d8c246527"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NSqZTVgVtb6oooUkgi01BC/g7IJU/gXpbiQfKLj09rQxnLy5jfJ8Wz29sTm7/9iEa13QOFi6N/fzLenWeQNMCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-10T14:51:14.145498Z","bundle_sha256":"e8e56f043c7d489c33916832740067532949246b73015c86dcef26e1a74e833c"}}