{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:JL5RKDICUQBJAFXXHOL72SV7ID","short_pith_number":"pith:JL5RKDIC","canonical_record":{"source":{"id":"2103.03098","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-03-01T22:39:49Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"27a4645364d0de9f49b7663bb976996657450ce524f4a58528af07900c8427fd","abstract_canon_sha256":"9a3fa58118b1397f9df18aaf8f09bb4b2c24f65dd58941492f6e8c7f66514849"},"schema_version":"1.0"},"canonical_sha256":"4afb150d02a4029016f73b97fd4abf40c1f04fe32f3e2fe2850490d743e54e37","source":{"kind":"arxiv","id":"2103.03098","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2103.03098","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"arxiv_version","alias_value":"2103.03098v1","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2103.03098","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_12","alias_value":"JL5RKDICUQBJ","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_16","alias_value":"JL5RKDICUQBJAFXX","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_8","alias_value":"JL5RKDIC","created_at":"2026-07-05T02:20:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:JL5RKDICUQBJAFXXHOL72SV7ID","target":"record","payload":{"canonical_record":{"source":{"id":"2103.03098","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-03-01T22:39:49Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"27a4645364d0de9f49b7663bb976996657450ce524f4a58528af07900c8427fd","abstract_canon_sha256":"9a3fa58118b1397f9df18aaf8f09bb4b2c24f65dd58941492f6e8c7f66514849"},"schema_version":"1.0"},"canonical_sha256":"4afb150d02a4029016f73b97fd4abf40c1f04fe32f3e2fe2850490d743e54e37","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:20:24.278733Z","signature_b64":"1r3OFRpu/JrJY1BP/2D4YgdjCcfaEUfuKjK5Ja1USlztBXPqSfuFeXLOTOxOWc38EJQ740lZGrd1P994ay2JCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4afb150d02a4029016f73b97fd4abf40c1f04fe32f3e2fe2850490d743e54e37","last_reissued_at":"2026-07-05T02:20:24.278242Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:20:24.278242Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2103.03098","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:20:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uJ9EOS8ylcc/Pwhu1AgkMGDBnvNryxg26hC2b6GL1w+Z74qLuPP/ckqhehtNS/qo/uxPMe6LY+Bw+MmZEoj8BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T17:15:10.909567Z"},"content_sha256":"f3d49b229b61adea218b06da0fb4819b310f0b86b2a0af85c46bdc4bd724886f","schema_version":"1.0","event_id":"sha256:f3d49b229b61adea218b06da0fb4819b310f0b86b2a0af85c46bdc4bd724886f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:JL5RKDICUQBJAFXXHOL72SV7ID","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Accounting for Variance in Machine Learning Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Assya Trofimov, Brennan Nichyporuk, Chris Pal, Dmitriy Serdyuk, Edward Raff, Ga\\\"el Varoquaux, Justin Szeto, Kanika Madan, Mirko Bronzi, Naz Sepah, Pascal Vincent, Pierre Delaunay, Samira Ebrahimi Kahou, Tal Arbel, Vikram Voleti, Vincent Michalski, Xavier Bouthillier","submitted_at":"2021-03-01T22:39:49Z","abstract_excerpt":"Strong empirical evidence that one machine-learning algorithm A outperforms another one B ideally calls for multiple trials optimizing the learning pipeline over sources of variation such as data sampling, data augmentation, parameter initialization, and hyperparameters choices. This is prohibitively expensive, and corners are cut to reach conclusions. We model the whole benchmarking process, revealing that variance due to data sampling, parameter initialization and hyperparameter choice impact markedly the results. We analyze the predominant comparison methods used today in the light of this "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2103.03098","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2103.03098/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:20:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xbKOoVtP2p/ukvWg9LTnjaeVEfFGzNoqM3r1r2lustedlORlVbgFFTVVJ+0TWmAMr7KDI3Ja1iSOnf359hwKBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T17:15:10.910128Z"},"content_sha256":"35548077d4fbdd57d2f67d7f1988892757676c9ae0ba73da642ddca861bd8e23","schema_version":"1.0","event_id":"sha256:35548077d4fbdd57d2f67d7f1988892757676c9ae0ba73da642ddca861bd8e23"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JL5RKDICUQBJAFXXHOL72SV7ID/bundle.json","state_url":"https://pith.science/pith/JL5RKDICUQBJAFXXHOL72SV7ID/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JL5RKDICUQBJAFXXHOL72SV7ID/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T17:15:10Z","links":{"resolver":"https://pith.science/pith/JL5RKDICUQBJAFXXHOL72SV7ID","bundle":"https://pith.science/pith/JL5RKDICUQBJAFXXHOL72SV7ID/bundle.json","state":"https://pith.science/pith/JL5RKDICUQBJAFXXHOL72SV7ID/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JL5RKDICUQBJAFXXHOL72SV7ID/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:JL5RKDICUQBJAFXXHOL72SV7ID","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9a3fa58118b1397f9df18aaf8f09bb4b2c24f65dd58941492f6e8c7f66514849","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-03-01T22:39:49Z","title_canon_sha256":"27a4645364d0de9f49b7663bb976996657450ce524f4a58528af07900c8427fd"},"schema_version":"1.0","source":{"id":"2103.03098","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2103.03098","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"arxiv_version","alias_value":"2103.03098v1","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2103.03098","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_12","alias_value":"JL5RKDICUQBJ","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_16","alias_value":"JL5RKDICUQBJAFXX","created_at":"2026-07-05T02:20:24Z"},{"alias_kind":"pith_short_8","alias_value":"JL5RKDIC","created_at":"2026-07-05T02:20:24Z"}],"graph_snapshots":[{"event_id":"sha256:35548077d4fbdd57d2f67d7f1988892757676c9ae0ba73da642ddca861bd8e23","target":"graph","created_at":"2026-07-05T02:20:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2103.03098/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Strong empirical evidence that one machine-learning algorithm A outperforms another one B ideally calls for multiple trials optimizing the learning pipeline over sources of variation such as data sampling, data augmentation, parameter initialization, and hyperparameters choices. This is prohibitively expensive, and corners are cut to reach conclusions. We model the whole benchmarking process, revealing that variance due to data sampling, parameter initialization and hyperparameter choice impact markedly the results. We analyze the predominant comparison methods used today in the light of this ","authors_text":"Assya Trofimov, Brennan Nichyporuk, Chris Pal, Dmitriy Serdyuk, Edward Raff, Ga\\\"el Varoquaux, Justin Szeto, Kanika Madan, Mirko Bronzi, Naz Sepah, Pascal Vincent, Pierre Delaunay, Samira Ebrahimi Kahou, Tal Arbel, Vikram Voleti, Vincent Michalski, Xavier Bouthillier","cross_cats":["stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-03-01T22:39:49Z","title":"Accounting for Variance in Machine Learning Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2103.03098","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f3d49b229b61adea218b06da0fb4819b310f0b86b2a0af85c46bdc4bd724886f","target":"record","created_at":"2026-07-05T02:20:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9a3fa58118b1397f9df18aaf8f09bb4b2c24f65dd58941492f6e8c7f66514849","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-03-01T22:39:49Z","title_canon_sha256":"27a4645364d0de9f49b7663bb976996657450ce524f4a58528af07900c8427fd"},"schema_version":"1.0","source":{"id":"2103.03098","kind":"arxiv","version":1}},"canonical_sha256":"4afb150d02a4029016f73b97fd4abf40c1f04fe32f3e2fe2850490d743e54e37","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4afb150d02a4029016f73b97fd4abf40c1f04fe32f3e2fe2850490d743e54e37","first_computed_at":"2026-07-05T02:20:24.278242Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:20:24.278242Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1r3OFRpu/JrJY1BP/2D4YgdjCcfaEUfuKjK5Ja1USlztBXPqSfuFeXLOTOxOWc38EJQ740lZGrd1P994ay2JCA==","signature_status":"signed_v1","signed_at":"2026-07-05T02:20:24.278733Z","signed_message":"canonical_sha256_bytes"},"source_id":"2103.03098","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f3d49b229b61adea218b06da0fb4819b310f0b86b2a0af85c46bdc4bd724886f","sha256:35548077d4fbdd57d2f67d7f1988892757676c9ae0ba73da642ddca861bd8e23"],"state_sha256":"5bb1624a004ba5a377f584cfc7ef3ae08875b01b2ab44f5d0a3a15f4c51be9e6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Sb/arTefPXPCK1eCOq+gDShmjJI29+3VyNTviJj89nYqqOPUzk4lww5UJidTRb5mnKfjSRg1WUALnJ5uwFPQCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T17:15:10.916402Z","bundle_sha256":"167d33b9096c62fae0bf0fdf0a5133fc37a2d4b859c395f5fe6fe339636f61b6"}}