{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:UNMASM224ZVJNNT7D6XRK7ZQEL","short_pith_number":"pith:UNMASM22","canonical_record":{"source":{"id":"2008.02637","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-08-06T13:17:43Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"58b4fc662ae73582f4ed5254fababd62a4f5f7c38194b7ea1ac7d32ecca27548","abstract_canon_sha256":"03b36503b34d5e034ec0cba3b1ebe4758cd21dbdb9d8481dbdc1dfaa04819a6d"},"schema_version":"1.0"},"canonical_sha256":"a35809335ae66a96b67f1faf157f3022fc90f9dcb2298f39dd6859276e9a9c1a","source":{"kind":"arxiv","id":"2008.02637","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2008.02637","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"arxiv_version","alias_value":"2008.02637v1","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2008.02637","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_12","alias_value":"UNMASM224ZVJ","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_16","alias_value":"UNMASM224ZVJNNT7","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_8","alias_value":"UNMASM22","created_at":"2026-07-05T01:25:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:UNMASM224ZVJNNT7D6XRK7ZQEL","target":"record","payload":{"canonical_record":{"source":{"id":"2008.02637","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-08-06T13:17:43Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"58b4fc662ae73582f4ed5254fababd62a4f5f7c38194b7ea1ac7d32ecca27548","abstract_canon_sha256":"03b36503b34d5e034ec0cba3b1ebe4758cd21dbdb9d8481dbdc1dfaa04819a6d"},"schema_version":"1.0"},"canonical_sha256":"a35809335ae66a96b67f1faf157f3022fc90f9dcb2298f39dd6859276e9a9c1a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:25:15.866354Z","signature_b64":"5UtMnR/G6HI4Z3ICKWqHit0GDoh+03ECLs59PzXM9BoDSLajtLhAygnBjkGC7Yhy1d0G7fd6msnbRX5ERUw1AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a35809335ae66a96b67f1faf157f3022fc90f9dcb2298f39dd6859276e9a9c1a","last_reissued_at":"2026-07-05T01:25:15.865982Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:25:15.865982Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2008.02637","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:25:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3H2R0k78CTBWijbNTiSVlhRJoK8gvsl0uCPgN9Ke5dKh7kmvNsGcr8wARIuCK5kwgGoRoafokiom3yWUWb/wDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-29T05:56:05.274450Z"},"content_sha256":"7ed3d4cb3eb4a1b59ee31898149d2b38e4cfcba3740e054a626a7d1bbe2d07af","schema_version":"1.0","event_id":"sha256:7ed3d4cb3eb4a1b59ee31898149d2b38e4cfcba3740e054a626a7d1bbe2d07af"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:UNMASM224ZVJNNT7D6XRK7ZQEL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Question and Answer Test-Train Overlap in Open-Domain Question Answering Datasets","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Patrick Lewis, Pontus Stenetorp, Sebastian Riedel","submitted_at":"2020-08-06T13:17:43Z","abstract_excerpt":"Ideally Open-Domain Question Answering models should exhibit a number of competencies, ranging from simply memorizing questions seen at training time, to answering novel question formulations with answers seen during training, to generalizing to completely novel questions with novel answers. However, single aggregated test set scores do not show the full picture of what capabilities models truly have. In this work, we perform a detailed study of the test sets of three popular open-domain benchmark datasets with respect to these competencies. We find that 60-70% of test-time answers are also pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2008.02637","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2008.02637/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:25:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pdmnk/Cx+HCIvEsYKAGTl3R/7uvMfSfXC9sxGn3CkkIkOpzSYk5Itjpe/v+zne7iLh43/Ol9SJN8im6sPqmMAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-29T05:56:05.274822Z"},"content_sha256":"6b33c84e380fb3db5f1c3d5878f450c759734640ff2bd898b9b9489b30bc4dfd","schema_version":"1.0","event_id":"sha256:6b33c84e380fb3db5f1c3d5878f450c759734640ff2bd898b9b9489b30bc4dfd"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/bundle.json","state_url":"https://pith.science/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-29T05:56:05Z","links":{"resolver":"https://pith.science/pith/UNMASM224ZVJNNT7D6XRK7ZQEL","bundle":"https://pith.science/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/bundle.json","state":"https://pith.science/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UNMASM224ZVJNNT7D6XRK7ZQEL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:UNMASM224ZVJNNT7D6XRK7ZQEL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"03b36503b34d5e034ec0cba3b1ebe4758cd21dbdb9d8481dbdc1dfaa04819a6d","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-08-06T13:17:43Z","title_canon_sha256":"58b4fc662ae73582f4ed5254fababd62a4f5f7c38194b7ea1ac7d32ecca27548"},"schema_version":"1.0","source":{"id":"2008.02637","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2008.02637","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"arxiv_version","alias_value":"2008.02637v1","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2008.02637","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_12","alias_value":"UNMASM224ZVJ","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_16","alias_value":"UNMASM224ZVJNNT7","created_at":"2026-07-05T01:25:15Z"},{"alias_kind":"pith_short_8","alias_value":"UNMASM22","created_at":"2026-07-05T01:25:15Z"}],"graph_snapshots":[{"event_id":"sha256:6b33c84e380fb3db5f1c3d5878f450c759734640ff2bd898b9b9489b30bc4dfd","target":"graph","created_at":"2026-07-05T01:25:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2008.02637/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Ideally Open-Domain Question Answering models should exhibit a number of competencies, ranging from simply memorizing questions seen at training time, to answering novel question formulations with answers seen during training, to generalizing to completely novel questions with novel answers. However, single aggregated test set scores do not show the full picture of what capabilities models truly have. In this work, we perform a detailed study of the test sets of three popular open-domain benchmark datasets with respect to these competencies. We find that 60-70% of test-time answers are also pr","authors_text":"Patrick Lewis, Pontus Stenetorp, Sebastian Riedel","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-08-06T13:17:43Z","title":"Question and Answer Test-Train Overlap in Open-Domain Question Answering Datasets"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2008.02637","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7ed3d4cb3eb4a1b59ee31898149d2b38e4cfcba3740e054a626a7d1bbe2d07af","target":"record","created_at":"2026-07-05T01:25:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"03b36503b34d5e034ec0cba3b1ebe4758cd21dbdb9d8481dbdc1dfaa04819a6d","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-08-06T13:17:43Z","title_canon_sha256":"58b4fc662ae73582f4ed5254fababd62a4f5f7c38194b7ea1ac7d32ecca27548"},"schema_version":"1.0","source":{"id":"2008.02637","kind":"arxiv","version":1}},"canonical_sha256":"a35809335ae66a96b67f1faf157f3022fc90f9dcb2298f39dd6859276e9a9c1a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a35809335ae66a96b67f1faf157f3022fc90f9dcb2298f39dd6859276e9a9c1a","first_computed_at":"2026-07-05T01:25:15.865982Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:25:15.865982Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"5UtMnR/G6HI4Z3ICKWqHit0GDoh+03ECLs59PzXM9BoDSLajtLhAygnBjkGC7Yhy1d0G7fd6msnbRX5ERUw1AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T01:25:15.866354Z","signed_message":"canonical_sha256_bytes"},"source_id":"2008.02637","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7ed3d4cb3eb4a1b59ee31898149d2b38e4cfcba3740e054a626a7d1bbe2d07af","sha256:6b33c84e380fb3db5f1c3d5878f450c759734640ff2bd898b9b9489b30bc4dfd"],"state_sha256":"204bd3e3de0673352993c70a39088e92383616b73c2cbbf7dafcb9a74b20f69c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HS/sbU/xDcWdAUwLrNGvp7I//v1J6cB+T8X4lTwl7wX/AKZfWur2h1v6HseuEt3AMJ5OAZLswj2gAhc5X2QfCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-29T05:56:05.277261Z","bundle_sha256":"ce7022faad6b6d3d9da61d70f587f3ed81289ffb72e3081bbbfe6d1922fecfe4"}}