{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:CCOAE7SZEZ4HGMCYZ4MBWTFXGN","short_pith_number":"pith:CCOAE7SZ","canonical_record":{"source":{"id":"2004.14444","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-29T19:34:19Z","cross_cats_sorted":["cs.CL","stat.ML"],"title_canon_sha256":"c08fcaf8dbcd785e6d8f591dffd473989e5d1c38bdac9bc8ccd9299a147453e8","abstract_canon_sha256":"0c775af04d7c170ebf1f2bbe7f9d2781f0d59519cb72c5606facc1737be92b9a"},"schema_version":"1.0"},"canonical_sha256":"109c027e592678733058cf181b4cb733718e3d165d9cd1684e6c7219c31bc63e","source":{"kind":"arxiv","id":"2004.14444","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.14444","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"arxiv_version","alias_value":"2004.14444v1","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.14444","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_12","alias_value":"CCOAE7SZEZ4H","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_16","alias_value":"CCOAE7SZEZ4HGMCY","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_8","alias_value":"CCOAE7SZ","created_at":"2026-07-05T00:59:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:CCOAE7SZEZ4HGMCYZ4MBWTFXGN","target":"record","payload":{"canonical_record":{"source":{"id":"2004.14444","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-29T19:34:19Z","cross_cats_sorted":["cs.CL","stat.ML"],"title_canon_sha256":"c08fcaf8dbcd785e6d8f591dffd473989e5d1c38bdac9bc8ccd9299a147453e8","abstract_canon_sha256":"0c775af04d7c170ebf1f2bbe7f9d2781f0d59519cb72c5606facc1737be92b9a"},"schema_version":"1.0"},"canonical_sha256":"109c027e592678733058cf181b4cb733718e3d165d9cd1684e6c7219c31bc63e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:59:26.372963Z","signature_b64":"m1lYZ5CZFl+323df6eX+Yn8XDju6SbhILCBLJkjoGAnCCo1oT2EqVYsNkcQzsNowezqpB5Qd4DOmv3mZK7/CBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"109c027e592678733058cf181b4cb733718e3d165d9cd1684e6c7219c31bc63e","last_reissued_at":"2026-07-05T00:59:26.372609Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:59:26.372609Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2004.14444","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:59:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DCn1BeLk0cni11o3JHu466nkkHE7oq9WZ6HkAlDP64cDLIlJZDIbnxkpLepmt1X11rLAL6+j2DhE8DyzfV+hDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-05T12:32:10.344541Z"},"content_sha256":"542b909edcc053f9030450ec4bd2ad3add8aa2fa13e75035670b5318302e9065","schema_version":"1.0","event_id":"sha256:542b909edcc053f9030450ec4bd2ad3add8aa2fa13e75035670b5318302e9065"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:CCOAE7SZEZ4HGMCYZ4MBWTFXGN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Effect of Natural Distribution Shift on Question Answering Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Benjamin Recht, John Miller, Karl Krauth, Ludwig Schmidt","submitted_at":"2020-04-29T19:34:19Z","abstract_excerpt":"We build four new test sets for the Stanford Question Answering Dataset (SQuAD) and evaluate the ability of question-answering systems to generalize to new data. Our first test set is from the original Wikipedia domain and measures the extent to which existing systems overfit the original test set. Despite several years of heavy test set re-use, we find no evidence of adaptive overfitting. The remaining three test sets are constructed from New York Times articles, Reddit posts, and Amazon product reviews and measure robustness to natural distribution shifts. Across a broad range of models, we "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.14444","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.14444/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:59:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AdsyTw/aVtBEtD4gi42qgVrDO3lgUcYeX8h0L0iZMNECqkRSZZ4D3KTBTsg1Xl3MAnK+FfPqwkA8/4DE8CoBDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-05T12:32:10.344915Z"},"content_sha256":"d35aed9d4819fe4ddecc51ed03ea0288839aeff4894d9ef23d3e4dd44df213bf","schema_version":"1.0","event_id":"sha256:d35aed9d4819fe4ddecc51ed03ea0288839aeff4894d9ef23d3e4dd44df213bf"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/bundle.json","state_url":"https://pith.science/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-05T12:32:10Z","links":{"resolver":"https://pith.science/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN","bundle":"https://pith.science/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/bundle.json","state":"https://pith.science/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CCOAE7SZEZ4HGMCYZ4MBWTFXGN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:CCOAE7SZEZ4HGMCYZ4MBWTFXGN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0c775af04d7c170ebf1f2bbe7f9d2781f0d59519cb72c5606facc1737be92b9a","cross_cats_sorted":["cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-29T19:34:19Z","title_canon_sha256":"c08fcaf8dbcd785e6d8f591dffd473989e5d1c38bdac9bc8ccd9299a147453e8"},"schema_version":"1.0","source":{"id":"2004.14444","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.14444","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"arxiv_version","alias_value":"2004.14444v1","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.14444","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_12","alias_value":"CCOAE7SZEZ4H","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_16","alias_value":"CCOAE7SZEZ4HGMCY","created_at":"2026-07-05T00:59:26Z"},{"alias_kind":"pith_short_8","alias_value":"CCOAE7SZ","created_at":"2026-07-05T00:59:26Z"}],"graph_snapshots":[{"event_id":"sha256:d35aed9d4819fe4ddecc51ed03ea0288839aeff4894d9ef23d3e4dd44df213bf","target":"graph","created_at":"2026-07-05T00:59:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.14444/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We build four new test sets for the Stanford Question Answering Dataset (SQuAD) and evaluate the ability of question-answering systems to generalize to new data. Our first test set is from the original Wikipedia domain and measures the extent to which existing systems overfit the original test set. Despite several years of heavy test set re-use, we find no evidence of adaptive overfitting. The remaining three test sets are constructed from New York Times articles, Reddit posts, and Amazon product reviews and measure robustness to natural distribution shifts. Across a broad range of models, we ","authors_text":"Benjamin Recht, John Miller, Karl Krauth, Ludwig Schmidt","cross_cats":["cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-29T19:34:19Z","title":"The Effect of Natural Distribution Shift on Question Answering Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.14444","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:542b909edcc053f9030450ec4bd2ad3add8aa2fa13e75035670b5318302e9065","target":"record","created_at":"2026-07-05T00:59:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0c775af04d7c170ebf1f2bbe7f9d2781f0d59519cb72c5606facc1737be92b9a","cross_cats_sorted":["cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-29T19:34:19Z","title_canon_sha256":"c08fcaf8dbcd785e6d8f591dffd473989e5d1c38bdac9bc8ccd9299a147453e8"},"schema_version":"1.0","source":{"id":"2004.14444","kind":"arxiv","version":1}},"canonical_sha256":"109c027e592678733058cf181b4cb733718e3d165d9cd1684e6c7219c31bc63e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"109c027e592678733058cf181b4cb733718e3d165d9cd1684e6c7219c31bc63e","first_computed_at":"2026-07-05T00:59:26.372609Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:59:26.372609Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"m1lYZ5CZFl+323df6eX+Yn8XDju6SbhILCBLJkjoGAnCCo1oT2EqVYsNkcQzsNowezqpB5Qd4DOmv3mZK7/CBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T00:59:26.372963Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.14444","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:542b909edcc053f9030450ec4bd2ad3add8aa2fa13e75035670b5318302e9065","sha256:d35aed9d4819fe4ddecc51ed03ea0288839aeff4894d9ef23d3e4dd44df213bf"],"state_sha256":"b08aa5f7a9aaec0fbc1a4b055976cd06745d4309d91a67f8de2b098d7a512219"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SJIdTZ23H+vjQsc4RcjPSNDzTlSavvusm2Uz7rsAEOgcb6qB8XXQ204BqGmhT/f+zfsqQ6alKTkKFPz+lNViBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-05T12:32:10.347070Z","bundle_sha256":"1b1d8cf282bf07b94d95ce28d857d9dfe87e236ddb6b8e5fcb3115ad4c44a232"}}