{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:H5D5J66J722RYQ734UHJKD6IVX","short_pith_number":"pith:H5D5J66J","canonical_record":{"source":{"id":"2212.01340","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2022-12-02T17:57:06Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"1aca05e82061d3a03a7e20b8f9d743107e7e6473ca73b02a85bdbb5e4caea251","abstract_canon_sha256":"a3547c6186fa7a094d470ab4351d90e29d179a5e91805f9bb4678f1e58057dac"},"schema_version":"1.0"},"canonical_sha256":"3f47d4fbc9feb51c43fbe50e950fc8addc572a2288f67cbc84f82ad3e5004c3f","source":{"kind":"arxiv","id":"2212.01340","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.01340","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"arxiv_version","alias_value":"2212.01340v1","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.01340","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_12","alias_value":"H5D5J66J722R","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_16","alias_value":"H5D5J66J722RYQ73","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_8","alias_value":"H5D5J66J","created_at":"2026-07-05T05:21:52Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:H5D5J66J722RYQ734UHJKD6IVX","target":"record","payload":{"canonical_record":{"source":{"id":"2212.01340","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2022-12-02T17:57:06Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"1aca05e82061d3a03a7e20b8f9d743107e7e6473ca73b02a85bdbb5e4caea251","abstract_canon_sha256":"a3547c6186fa7a094d470ab4351d90e29d179a5e91805f9bb4678f1e58057dac"},"schema_version":"1.0"},"canonical_sha256":"3f47d4fbc9feb51c43fbe50e950fc8addc572a2288f67cbc84f82ad3e5004c3f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:21:52.147672Z","signature_b64":"yAATg3EAwZJNgluXmP7LeXuPODmoY/RMCjy4OxagfvVZt2lfIVVEzFB6eCJuPKyd/Ss6fiXLvmne/M3zaEvKCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3f47d4fbc9feb51c43fbe50e950fc8addc572a2288f67cbc84f82ad3e5004c3f","last_reissued_at":"2026-07-05T05:21:52.147172Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:21:52.147172Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2212.01340","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:21:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NdJFeBOqy1XN+5ngxvleCkwrv8qoI20LIVi3twxYrT0cY9u6M2v5kxsKT9qJwc8Fyg+pCTX+M12LlO/RMSTaCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:40:50.333364Z"},"content_sha256":"5304d327f05db36107783ddc32b619685bcdd74b26e83d8c686df7a5635d0c04","schema_version":"1.0","event_id":"sha256:5304d327f05db36107783ddc32b619685bcdd74b26e83d8c686df7a5635d0c04"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:H5D5J66J722RYQ734UHJKD6IVX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Moving Beyond Downstream Task Accuracy for Information Retrieval Benchmarking","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Avirup Sil, Christopher Potts, Jon Saad-Falcon, Keshav Santhanam, Martin Franz, Matei Zaharia, Md Arafat Sultan, Omar Khattab, Radu Florian, Salim Roukos","submitted_at":"2022-12-02T17:57:06Z","abstract_excerpt":"Neural information retrieval (IR) systems have progressed rapidly in recent years, in large part due to the release of publicly available benchmarking tasks. Unfortunately, some dimensions of this progress are illusory: the majority of the popular IR benchmarks today focus exclusively on downstream task accuracy and thus conceal the costs incurred by systems that trade away efficiency for quality. Latency, hardware cost, and other efficiency considerations are paramount to the deployment of IR systems in user-facing settings. We propose that IR benchmarks structure their evaluation methodology"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.01340","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.01340/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:21:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CNqIPHs9daIANo43qz+zkz5hwz1G+kXfdgyOGFvxJ+EVgbWch/COmw2l1BTJW/TRel4pvMQPVQmkPxr9zUZ7AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:40:50.333739Z"},"content_sha256":"8b3f9d5b529961e79d187cd389d9aeb2a52045fbbbf4b92f7caaa8623cb5e392","schema_version":"1.0","event_id":"sha256:8b3f9d5b529961e79d187cd389d9aeb2a52045fbbbf4b92f7caaa8623cb5e392"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/H5D5J66J722RYQ734UHJKD6IVX/bundle.json","state_url":"https://pith.science/pith/H5D5J66J722RYQ734UHJKD6IVX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/H5D5J66J722RYQ734UHJKD6IVX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T11:40:50Z","links":{"resolver":"https://pith.science/pith/H5D5J66J722RYQ734UHJKD6IVX","bundle":"https://pith.science/pith/H5D5J66J722RYQ734UHJKD6IVX/bundle.json","state":"https://pith.science/pith/H5D5J66J722RYQ734UHJKD6IVX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/H5D5J66J722RYQ734UHJKD6IVX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:H5D5J66J722RYQ734UHJKD6IVX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a3547c6186fa7a094d470ab4351d90e29d179a5e91805f9bb4678f1e58057dac","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2022-12-02T17:57:06Z","title_canon_sha256":"1aca05e82061d3a03a7e20b8f9d743107e7e6473ca73b02a85bdbb5e4caea251"},"schema_version":"1.0","source":{"id":"2212.01340","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.01340","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"arxiv_version","alias_value":"2212.01340v1","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.01340","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_12","alias_value":"H5D5J66J722R","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_16","alias_value":"H5D5J66J722RYQ73","created_at":"2026-07-05T05:21:52Z"},{"alias_kind":"pith_short_8","alias_value":"H5D5J66J","created_at":"2026-07-05T05:21:52Z"}],"graph_snapshots":[{"event_id":"sha256:8b3f9d5b529961e79d187cd389d9aeb2a52045fbbbf4b92f7caaa8623cb5e392","target":"graph","created_at":"2026-07-05T05:21:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2212.01340/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Neural information retrieval (IR) systems have progressed rapidly in recent years, in large part due to the release of publicly available benchmarking tasks. Unfortunately, some dimensions of this progress are illusory: the majority of the popular IR benchmarks today focus exclusively on downstream task accuracy and thus conceal the costs incurred by systems that trade away efficiency for quality. Latency, hardware cost, and other efficiency considerations are paramount to the deployment of IR systems in user-facing settings. We propose that IR benchmarks structure their evaluation methodology","authors_text":"Avirup Sil, Christopher Potts, Jon Saad-Falcon, Keshav Santhanam, Martin Franz, Matei Zaharia, Md Arafat Sultan, Omar Khattab, Radu Florian, Salim Roukos","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2022-12-02T17:57:06Z","title":"Moving Beyond Downstream Task Accuracy for Information Retrieval Benchmarking"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.01340","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5304d327f05db36107783ddc32b619685bcdd74b26e83d8c686df7a5635d0c04","target":"record","created_at":"2026-07-05T05:21:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a3547c6186fa7a094d470ab4351d90e29d179a5e91805f9bb4678f1e58057dac","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2022-12-02T17:57:06Z","title_canon_sha256":"1aca05e82061d3a03a7e20b8f9d743107e7e6473ca73b02a85bdbb5e4caea251"},"schema_version":"1.0","source":{"id":"2212.01340","kind":"arxiv","version":1}},"canonical_sha256":"3f47d4fbc9feb51c43fbe50e950fc8addc572a2288f67cbc84f82ad3e5004c3f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3f47d4fbc9feb51c43fbe50e950fc8addc572a2288f67cbc84f82ad3e5004c3f","first_computed_at":"2026-07-05T05:21:52.147172Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:21:52.147172Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yAATg3EAwZJNgluXmP7LeXuPODmoY/RMCjy4OxagfvVZt2lfIVVEzFB6eCJuPKyd/Ss6fiXLvmne/M3zaEvKCg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:21:52.147672Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.01340","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5304d327f05db36107783ddc32b619685bcdd74b26e83d8c686df7a5635d0c04","sha256:8b3f9d5b529961e79d187cd389d9aeb2a52045fbbbf4b92f7caaa8623cb5e392"],"state_sha256":"699b110740e8f6d2df59c5f79d8a44467b62d9e38928cd0abf0df3fd838dfe6b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jRXePPNtJV3R0ymhkm0W6XvW8ViWSxERQHUdfXy3+WxcVoeKRg/aV8b9M+hCMXVqR0OBQQpLcWP6SqEEKyEjAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T11:40:50.335710Z","bundle_sha256":"18ff92723d95bed9472529a7d54af4f18068fd06a1ed61337ec988f3e0bb402c"}}