{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2015:BS5WGYSQBUDEZ3W2SLAYZKEH2F","short_pith_number":"pith:BS5WGYSQ","canonical_record":{"source":{"id":"1502.04585","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2015-02-16T15:53:03Z","cross_cats_sorted":[],"title_canon_sha256":"d9228420d265379d5e411ceda76f7b05c916af1f059cc9d3d65c9be454c36f24","abstract_canon_sha256":"336015f918673807c8d68f21238f2e20ebe32258aef063a6674c4f37cebfa519"},"schema_version":"1.0"},"canonical_sha256":"0cbb6362500d064ceeda92c18ca887d16a7973d4b0805f8c88f009652702fbea","source":{"kind":"arxiv","id":"1502.04585","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1502.04585","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"arxiv_version","alias_value":"1502.04585v1","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1502.04585","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"pith_short_12","alias_value":"BS5WGYSQBUDE","created_at":"2026-05-18T12:29:14Z"},{"alias_kind":"pith_short_16","alias_value":"BS5WGYSQBUDEZ3W2","created_at":"2026-05-18T12:29:14Z"},{"alias_kind":"pith_short_8","alias_value":"BS5WGYSQ","created_at":"2026-05-18T12:29:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2015:BS5WGYSQBUDEZ3W2SLAYZKEH2F","target":"record","payload":{"canonical_record":{"source":{"id":"1502.04585","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2015-02-16T15:53:03Z","cross_cats_sorted":[],"title_canon_sha256":"d9228420d265379d5e411ceda76f7b05c916af1f059cc9d3d65c9be454c36f24","abstract_canon_sha256":"336015f918673807c8d68f21238f2e20ebe32258aef063a6674c4f37cebfa519"},"schema_version":"1.0"},"canonical_sha256":"0cbb6362500d064ceeda92c18ca887d16a7973d4b0805f8c88f009652702fbea","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T02:26:59.530070Z","signature_b64":"cgYQnT8TuPQXV/8LgcalIkIZsiqyjUjnj5xHOz2L6l5+fSew6+DaYfLVlIXZgli+FHncRjbQvY4T6jx1iq/lDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0cbb6362500d064ceeda92c18ca887d16a7973d4b0805f8c88f009652702fbea","last_reissued_at":"2026-05-18T02:26:59.529700Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T02:26:59.529700Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1502.04585","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T02:26:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ni85OxbYBKK8VYKs4xYAGzD9J16zxr6XuymxTej20ccbP1JPEHXLvyqSH5IPeKMa4mi+bSb0QMzMnTh1nM1uBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-21T19:25:52.689792Z"},"content_sha256":"a268dc121620147ba61a1a2229c5887832136333fdafdb62ab07b3417ec557da","schema_version":"1.0","event_id":"sha256:a268dc121620147ba61a1a2229c5887832136333fdafdb62ab07b3417ec557da"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2015:BS5WGYSQBUDEZ3W2SLAYZKEH2F","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Ladder: A Reliable Leaderboard for Machine Learning Competitions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Avrim Blum, Moritz Hardt","submitted_at":"2015-02-16T15:53:03Z","abstract_excerpt":"The organizer of a machine learning competition faces the problem of maintaining an accurate leaderboard that faithfully represents the quality of the best submission of each competing team. What makes this estimation problem particularly challenging is its sequential and adaptive nature. As participants are allowed to repeatedly evaluate their submissions on the leaderboard, they may begin to overfit to the holdout data that supports the leaderboard. Few theoretical results give actionable advice on how to design a reliable leaderboard. Existing approaches therefore often resort to poorly und"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1502.04585","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T02:26:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uIn7/NSR793DYO7jEy0lxQ9/L3M81/wsCyc3pezF8k5kwL5tX9aojJJsltZeNCxMNBHYKGvzS5vKVc67afxJBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-21T19:25:52.690137Z"},"content_sha256":"e54b9e98b0049948f863a23527b16f620b2ac77259df496426f395269b5696d8","schema_version":"1.0","event_id":"sha256:e54b9e98b0049948f863a23527b16f620b2ac77259df496426f395269b5696d8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/bundle.json","state_url":"https://pith.science/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-21T19:25:52Z","links":{"resolver":"https://pith.science/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F","bundle":"https://pith.science/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/bundle.json","state":"https://pith.science/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BS5WGYSQBUDEZ3W2SLAYZKEH2F/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2015:BS5WGYSQBUDEZ3W2SLAYZKEH2F","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"336015f918673807c8d68f21238f2e20ebe32258aef063a6674c4f37cebfa519","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2015-02-16T15:53:03Z","title_canon_sha256":"d9228420d265379d5e411ceda76f7b05c916af1f059cc9d3d65c9be454c36f24"},"schema_version":"1.0","source":{"id":"1502.04585","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1502.04585","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"arxiv_version","alias_value":"1502.04585v1","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1502.04585","created_at":"2026-05-18T02:26:59Z"},{"alias_kind":"pith_short_12","alias_value":"BS5WGYSQBUDE","created_at":"2026-05-18T12:29:14Z"},{"alias_kind":"pith_short_16","alias_value":"BS5WGYSQBUDEZ3W2","created_at":"2026-05-18T12:29:14Z"},{"alias_kind":"pith_short_8","alias_value":"BS5WGYSQ","created_at":"2026-05-18T12:29:14Z"}],"graph_snapshots":[{"event_id":"sha256:e54b9e98b0049948f863a23527b16f620b2ac77259df496426f395269b5696d8","target":"graph","created_at":"2026-05-18T02:26:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"The organizer of a machine learning competition faces the problem of maintaining an accurate leaderboard that faithfully represents the quality of the best submission of each competing team. What makes this estimation problem particularly challenging is its sequential and adaptive nature. As participants are allowed to repeatedly evaluate their submissions on the leaderboard, they may begin to overfit to the holdout data that supports the leaderboard. Few theoretical results give actionable advice on how to design a reliable leaderboard. Existing approaches therefore often resort to poorly und","authors_text":"Avrim Blum, Moritz Hardt","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2015-02-16T15:53:03Z","title":"The Ladder: A Reliable Leaderboard for Machine Learning Competitions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1502.04585","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a268dc121620147ba61a1a2229c5887832136333fdafdb62ab07b3417ec557da","target":"record","created_at":"2026-05-18T02:26:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"336015f918673807c8d68f21238f2e20ebe32258aef063a6674c4f37cebfa519","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2015-02-16T15:53:03Z","title_canon_sha256":"d9228420d265379d5e411ceda76f7b05c916af1f059cc9d3d65c9be454c36f24"},"schema_version":"1.0","source":{"id":"1502.04585","kind":"arxiv","version":1}},"canonical_sha256":"0cbb6362500d064ceeda92c18ca887d16a7973d4b0805f8c88f009652702fbea","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0cbb6362500d064ceeda92c18ca887d16a7973d4b0805f8c88f009652702fbea","first_computed_at":"2026-05-18T02:26:59.529700Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T02:26:59.529700Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cgYQnT8TuPQXV/8LgcalIkIZsiqyjUjnj5xHOz2L6l5+fSew6+DaYfLVlIXZgli+FHncRjbQvY4T6jx1iq/lDg==","signature_status":"signed_v1","signed_at":"2026-05-18T02:26:59.530070Z","signed_message":"canonical_sha256_bytes"},"source_id":"1502.04585","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a268dc121620147ba61a1a2229c5887832136333fdafdb62ab07b3417ec557da","sha256:e54b9e98b0049948f863a23527b16f620b2ac77259df496426f395269b5696d8"],"state_sha256":"d9cdca724c14e450cd06442b0031ab346828666c24766ca3bdd1eef020a5e0e2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ToXyIyNgeD33AWdr4XqgxIGkrUEPs35UrTIMgu5k2yINBloygrdIov7N47GHUQYffGcsQtQCMka4dvQHQPMiCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-21T19:25:52.692343Z","bundle_sha256":"ecf7c8beab98975e8ff97b15058f67e675244438664f07d3bd9f9b48f87599c7"}}