{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:ELFR6MBZOJN6WHVC5ZCJY7BTFL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5f997755dd855605f34f8fca756e81d51e1406a935df8b76e367d5887a23f943","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2021-02-25T14:42:55Z","title_canon_sha256":"6e3e15d28349ef17a75c6c75dd12fa71d7baa9bdc7044067549a3a59363311bf"},"schema_version":"1.0","source":{"id":"2102.12887","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.12887","created_at":"2026-07-05T02:18:24Z"},{"alias_kind":"arxiv_version","alias_value":"2102.12887v1","created_at":"2026-07-05T02:18:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.12887","created_at":"2026-07-05T02:18:24Z"},{"alias_kind":"pith_short_12","alias_value":"ELFR6MBZOJN6","created_at":"2026-07-05T02:18:24Z"},{"alias_kind":"pith_short_16","alias_value":"ELFR6MBZOJN6WHVC","created_at":"2026-07-05T02:18:24Z"},{"alias_kind":"pith_short_8","alias_value":"ELFR6MBZ","created_at":"2026-07-05T02:18:24Z"}],"graph_snapshots":[{"event_id":"sha256:0b4830f7bd49907df3010ff4993566a73d13ddb881e74899bcd77bbf1010f29e","target":"graph","created_at":"2026-07-05T02:18:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2102.12887/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Leaderboards are a ubiquitous part of modern research in applied machine learning. By design, they sort entries into some linear order, where the top-scoring entry is recognized as the \"state of the art\" (SOTA). Due to the rapid progress being made in information retrieval today, particularly with neural models, the top entry in a leaderboard is replaced with some regularity. These are touted as improvements in the state of the art. Such pronouncements, however, are almost never qualified with significance testing. In the context of the MS MARCO document ranking leaderboard, we pose a specific","authors_text":"Bhaskar Mitra, Daniel Campos, Emine Yilmaz, Jimmy Lin, Nick Craswell","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2021-02-25T14:42:55Z","title":"Significant Improvements over the State of the Art? A Case Study of the MS MARCO Document Ranking Leaderboard"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.12887","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:93d1bbd5ce0f6fd87e0dc2f44334d1386018d4941e297864de0a623ae85e9be3","target":"record","created_at":"2026-07-05T02:18:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5f997755dd855605f34f8fca756e81d51e1406a935df8b76e367d5887a23f943","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2021-02-25T14:42:55Z","title_canon_sha256":"6e3e15d28349ef17a75c6c75dd12fa71d7baa9bdc7044067549a3a59363311bf"},"schema_version":"1.0","source":{"id":"2102.12887","kind":"arxiv","version":1}},"canonical_sha256":"22cb1f3039725beb1ea2ee449c7c332adc3744a5dd1714e6f29f87c4a03de67a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"22cb1f3039725beb1ea2ee449c7c332adc3744a5dd1714e6f29f87c4a03de67a","first_computed_at":"2026-07-05T02:18:24.474336Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:18:24.474336Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zAvQXBsI6pzS2l3d6picwtQDhPrS1nW6W7KCzltLqN6dbFFPxTulH+dp8/9aX49HFFEwzm0Rc3Qs+MK9d6mVDw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:18:24.474721Z","signed_message":"canonical_sha256_bytes"},"source_id":"2102.12887","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:93d1bbd5ce0f6fd87e0dc2f44334d1386018d4941e297864de0a623ae85e9be3","sha256:0b4830f7bd49907df3010ff4993566a73d13ddb881e74899bcd77bbf1010f29e"],"state_sha256":"1558a54a177fbeb0cd89a0d51e6fa7354594df745fa9dd9788b62b806eca46b6"}