{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:UGACXKB6P3Y6XSTGS65ZHRW2L6","short_pith_number":"pith:UGACXKB6","canonical_record":{"source":{"id":"2206.14759","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2022-06-29T16:44:43Z","cross_cats_sorted":[],"title_canon_sha256":"1d90e9bb7cd348a5de722659747f3b443f0cee1473bfb9c0270af67bfb5e8d3b","abstract_canon_sha256":"4f6ed7756cde8a44ead5762bfdc190a0c4ed08a0323b5b9a77275ff5bf37e655"},"schema_version":"1.0"},"canonical_sha256":"a1802ba83e7ef1ebca6697bb93c6da5fb6409f7d27b06b82a7614be7aa2768db","source":{"kind":"arxiv","id":"2206.14759","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.14759","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"arxiv_version","alias_value":"2206.14759v2","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.14759","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_12","alias_value":"UGACXKB6P3Y6","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_16","alias_value":"UGACXKB6P3Y6XSTG","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_8","alias_value":"UGACXKB6","created_at":"2026-07-05T04:52:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:UGACXKB6P3Y6XSTGS65ZHRW2L6","target":"record","payload":{"canonical_record":{"source":{"id":"2206.14759","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2022-06-29T16:44:43Z","cross_cats_sorted":[],"title_canon_sha256":"1d90e9bb7cd348a5de722659747f3b443f0cee1473bfb9c0270af67bfb5e8d3b","abstract_canon_sha256":"4f6ed7756cde8a44ead5762bfdc190a0c4ed08a0323b5b9a77275ff5bf37e655"},"schema_version":"1.0"},"canonical_sha256":"a1802ba83e7ef1ebca6697bb93c6da5fb6409f7d27b06b82a7614be7aa2768db","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:52:58.976330Z","signature_b64":"K+W1YWko5RJHCZ/DzkCV93uDXyuw582oHjg4NTU+R2vu/Aw+NVy9Dua4yVV4TQLqD0TmBlRed2QqYxkhoJntBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a1802ba83e7ef1ebca6697bb93c6da5fb6409f7d27b06b82a7614be7aa2768db","last_reissued_at":"2026-07-05T04:52:58.975881Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:52:58.975881Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2206.14759","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:52:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"65nvCKs3bHHRuxqykw5Fi6S3dCbarW7t9NnK1U1rDOMhTbmMsKkLyetSjmqutcqOW2svII8yuJKbekqYnpAVBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T09:20:01.143674Z"},"content_sha256":"2b77b5b95acd73312fd1be723382656fd5f0a996a2fc9ee591479fda95fd830b","schema_version":"1.0","event_id":"sha256:2b77b5b95acd73312fd1be723382656fd5f0a996a2fc9ee591479fda95fd830b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:UGACXKB6P3Y6XSTGS65ZHRW2L6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How Train-Test Leakage Affects Zero-shot Retrieval","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Christopher Akiki, Maik Fr\\\"obe, Martin Potthast, Matthias Hagen","submitted_at":"2022-06-29T16:44:43Z","abstract_excerpt":"Neural retrieval models are often trained on (subsets of) the millions of queries of the MS MARCO / ORCAS datasets and then tested on the 250 Robust04 queries or other TREC benchmarks with often only 50 queries. In such setups, many of the few test queries can be very similar to queries from the huge training data -- in fact, 69% of the Robust04 queries have near-duplicates in MS MARCO / ORCAS. We investigate the impact of this unintended train-test leakage by training neural retrieval models on combinations of a fixed number of MS MARCO / ORCAS queries that are highly similar to the actual te"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.14759","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2206.14759/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:52:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pO2jx/5KrwDa6Ar/JfOVOtCIYngyqTmfLofNzCvRglMuhagdrRUSvZ3woPg9tJKMTMrBUPX3Dme/OUZ8lb/FAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T09:20:01.144158Z"},"content_sha256":"48f9d16f06f255c17e65bfdd8605d18c7eb61b6797340d2cb0bc3b7dfe20ae54","schema_version":"1.0","event_id":"sha256:48f9d16f06f255c17e65bfdd8605d18c7eb61b6797340d2cb0bc3b7dfe20ae54"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/bundle.json","state_url":"https://pith.science/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T09:20:01Z","links":{"resolver":"https://pith.science/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6","bundle":"https://pith.science/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/bundle.json","state":"https://pith.science/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UGACXKB6P3Y6XSTGS65ZHRW2L6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:UGACXKB6P3Y6XSTGS65ZHRW2L6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4f6ed7756cde8a44ead5762bfdc190a0c4ed08a0323b5b9a77275ff5bf37e655","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2022-06-29T16:44:43Z","title_canon_sha256":"1d90e9bb7cd348a5de722659747f3b443f0cee1473bfb9c0270af67bfb5e8d3b"},"schema_version":"1.0","source":{"id":"2206.14759","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.14759","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"arxiv_version","alias_value":"2206.14759v2","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.14759","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_12","alias_value":"UGACXKB6P3Y6","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_16","alias_value":"UGACXKB6P3Y6XSTG","created_at":"2026-07-05T04:52:58Z"},{"alias_kind":"pith_short_8","alias_value":"UGACXKB6","created_at":"2026-07-05T04:52:58Z"}],"graph_snapshots":[{"event_id":"sha256:48f9d16f06f255c17e65bfdd8605d18c7eb61b6797340d2cb0bc3b7dfe20ae54","target":"graph","created_at":"2026-07-05T04:52:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2206.14759/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Neural retrieval models are often trained on (subsets of) the millions of queries of the MS MARCO / ORCAS datasets and then tested on the 250 Robust04 queries or other TREC benchmarks with often only 50 queries. In such setups, many of the few test queries can be very similar to queries from the huge training data -- in fact, 69% of the Robust04 queries have near-duplicates in MS MARCO / ORCAS. We investigate the impact of this unintended train-test leakage by training neural retrieval models on combinations of a fixed number of MS MARCO / ORCAS queries that are highly similar to the actual te","authors_text":"Christopher Akiki, Maik Fr\\\"obe, Martin Potthast, Matthias Hagen","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2022-06-29T16:44:43Z","title":"How Train-Test Leakage Affects Zero-shot Retrieval"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.14759","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2b77b5b95acd73312fd1be723382656fd5f0a996a2fc9ee591479fda95fd830b","target":"record","created_at":"2026-07-05T04:52:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4f6ed7756cde8a44ead5762bfdc190a0c4ed08a0323b5b9a77275ff5bf37e655","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2022-06-29T16:44:43Z","title_canon_sha256":"1d90e9bb7cd348a5de722659747f3b443f0cee1473bfb9c0270af67bfb5e8d3b"},"schema_version":"1.0","source":{"id":"2206.14759","kind":"arxiv","version":2}},"canonical_sha256":"a1802ba83e7ef1ebca6697bb93c6da5fb6409f7d27b06b82a7614be7aa2768db","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a1802ba83e7ef1ebca6697bb93c6da5fb6409f7d27b06b82a7614be7aa2768db","first_computed_at":"2026-07-05T04:52:58.975881Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:52:58.975881Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"K+W1YWko5RJHCZ/DzkCV93uDXyuw582oHjg4NTU+R2vu/Aw+NVy9Dua4yVV4TQLqD0TmBlRed2QqYxkhoJntBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:52:58.976330Z","signed_message":"canonical_sha256_bytes"},"source_id":"2206.14759","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2b77b5b95acd73312fd1be723382656fd5f0a996a2fc9ee591479fda95fd830b","sha256:48f9d16f06f255c17e65bfdd8605d18c7eb61b6797340d2cb0bc3b7dfe20ae54"],"state_sha256":"2060430bf824c9e3aef07ac89077fea1f580105b6b60cffcd29ba84226b7373a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6NhvQaHQ6YmDq7Bbx/Rd0rzFC1w2lsVFqtmzQRMduBaLYLgc5pozKhWE4Ju8S5wQI1AKj2RjcdsDdbTIKMqBCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T09:20:01.148186Z","bundle_sha256":"b31685371dcec3176ca9cd10d2f193ca4f2454bb3afe71585db9439dca142c85"}}