{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:4QXK2UJKC6WVNMCOTJOGSYBDXA","short_pith_number":"pith:4QXK2UJK","canonical_record":{"source":{"id":"2410.08385","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-10T21:44:56Z","cross_cats_sorted":["cs.AI","cs.CY","cs.SE"],"title_canon_sha256":"07dc3d8d7be5b401e5cb1ff78273cc6e531856c1f5b643afef6ff84fddd19f13","abstract_canon_sha256":"c3085b37fd9544908a08781c285ad95b32727ec365ce005e313dfa6def46e438"},"schema_version":"1.0"},"canonical_sha256":"e42ead512a17ad56b04e9a5c696023b836869e7fc043a78987d73769f5e42982","source":{"kind":"arxiv","id":"2410.08385","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.08385","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"arxiv_version","alias_value":"2410.08385v2","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.08385","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_12","alias_value":"4QXK2UJKC6WV","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_16","alias_value":"4QXK2UJKC6WVNMCO","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_8","alias_value":"4QXK2UJK","created_at":"2026-07-05T11:41:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:4QXK2UJKC6WVNMCOTJOGSYBDXA","target":"record","payload":{"canonical_record":{"source":{"id":"2410.08385","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-10T21:44:56Z","cross_cats_sorted":["cs.AI","cs.CY","cs.SE"],"title_canon_sha256":"07dc3d8d7be5b401e5cb1ff78273cc6e531856c1f5b643afef6ff84fddd19f13","abstract_canon_sha256":"c3085b37fd9544908a08781c285ad95b32727ec365ce005e313dfa6def46e438"},"schema_version":"1.0"},"canonical_sha256":"e42ead512a17ad56b04e9a5c696023b836869e7fc043a78987d73769f5e42982","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:41:35.930495Z","signature_b64":"v6Gduq0vRD7FM5TPTW0f6k6/ux5TiT1HrvmiEPDU9VBNvkiF/zQc2NLD7a/IKN58j7QUSWpU8ip/90mcnnHBBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e42ead512a17ad56b04e9a5c696023b836869e7fc043a78987d73769f5e42982","last_reissued_at":"2026-07-05T11:41:35.929971Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:41:35.929971Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.08385","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:41:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GZ30eYiQKGZ3Av8ZjBgi6ZnNcKbm5qjMyXeq6BqbRCk6nPU+Aifz+lJK2vDbrOSidv5wAbMj8y6cA3YG/uLtDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T11:16:20.104862Z"},"content_sha256":"1796cbb124a12c1e0397a75ce4e440902c4607948872ff116ca421096bf82c94","schema_version":"1.0","event_id":"sha256:1796cbb124a12c1e0397a75ce4e440902c4607948872ff116ca421096bf82c94"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:4QXK2UJKC6WVNMCOTJOGSYBDXA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Language model developers should report train-test overlap","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CY","cs.SE"],"primary_cat":"cs.LG","authors_text":"Andy K Zhang, Kevin Klyman, Percy Liang, Rishi Bommasani, Yian Zhang, Yifan Mai, Yoav Levine","submitted_at":"2024-10-10T21:44:56Z","abstract_excerpt":"Language models are extensively evaluated, but correctly interpreting evaluation results requires knowledge of train-test overlap which refers to the extent to which the language model is trained on the very data it is being tested on. The public currently lacks adequate information about train-test overlap: most models have no public train-test overlap statistics, and third parties cannot directly measure train-test overlap since they do not have access to the training data. To make this clear, we document the practices of 30 model developers, finding that just 9 developers report train-test "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.08385","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.08385/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:41:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EmYg1dTiIXv29ani4HafnAQ/I8TDUa5Bxn613kv+ABDM62hLYfCMDKcVSd+UrQEbNdbXD3LkUDI7uk9HouAiCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T11:16:20.105393Z"},"content_sha256":"f776124b6b6038ebea08f85740cdc8035749cce943537d7b9dbc209b244ba846","schema_version":"1.0","event_id":"sha256:f776124b6b6038ebea08f85740cdc8035749cce943537d7b9dbc209b244ba846"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/bundle.json","state_url":"https://pith.science/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T11:16:20Z","links":{"resolver":"https://pith.science/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA","bundle":"https://pith.science/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/bundle.json","state":"https://pith.science/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4QXK2UJKC6WVNMCOTJOGSYBDXA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:4QXK2UJKC6WVNMCOTJOGSYBDXA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c3085b37fd9544908a08781c285ad95b32727ec365ce005e313dfa6def46e438","cross_cats_sorted":["cs.AI","cs.CY","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-10T21:44:56Z","title_canon_sha256":"07dc3d8d7be5b401e5cb1ff78273cc6e531856c1f5b643afef6ff84fddd19f13"},"schema_version":"1.0","source":{"id":"2410.08385","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.08385","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"arxiv_version","alias_value":"2410.08385v2","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.08385","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_12","alias_value":"4QXK2UJKC6WV","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_16","alias_value":"4QXK2UJKC6WVNMCO","created_at":"2026-07-05T11:41:35Z"},{"alias_kind":"pith_short_8","alias_value":"4QXK2UJK","created_at":"2026-07-05T11:41:35Z"}],"graph_snapshots":[{"event_id":"sha256:f776124b6b6038ebea08f85740cdc8035749cce943537d7b9dbc209b244ba846","target":"graph","created_at":"2026-07-05T11:41:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.08385/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language models are extensively evaluated, but correctly interpreting evaluation results requires knowledge of train-test overlap which refers to the extent to which the language model is trained on the very data it is being tested on. The public currently lacks adequate information about train-test overlap: most models have no public train-test overlap statistics, and third parties cannot directly measure train-test overlap since they do not have access to the training data. To make this clear, we document the practices of 30 model developers, finding that just 9 developers report train-test ","authors_text":"Andy K Zhang, Kevin Klyman, Percy Liang, Rishi Bommasani, Yian Zhang, Yifan Mai, Yoav Levine","cross_cats":["cs.AI","cs.CY","cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-10T21:44:56Z","title":"Language model developers should report train-test overlap"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.08385","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1796cbb124a12c1e0397a75ce4e440902c4607948872ff116ca421096bf82c94","target":"record","created_at":"2026-07-05T11:41:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c3085b37fd9544908a08781c285ad95b32727ec365ce005e313dfa6def46e438","cross_cats_sorted":["cs.AI","cs.CY","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-10T21:44:56Z","title_canon_sha256":"07dc3d8d7be5b401e5cb1ff78273cc6e531856c1f5b643afef6ff84fddd19f13"},"schema_version":"1.0","source":{"id":"2410.08385","kind":"arxiv","version":2}},"canonical_sha256":"e42ead512a17ad56b04e9a5c696023b836869e7fc043a78987d73769f5e42982","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e42ead512a17ad56b04e9a5c696023b836869e7fc043a78987d73769f5e42982","first_computed_at":"2026-07-05T11:41:35.929971Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:41:35.929971Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"v6Gduq0vRD7FM5TPTW0f6k6/ux5TiT1HrvmiEPDU9VBNvkiF/zQc2NLD7a/IKN58j7QUSWpU8ip/90mcnnHBBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:41:35.930495Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.08385","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1796cbb124a12c1e0397a75ce4e440902c4607948872ff116ca421096bf82c94","sha256:f776124b6b6038ebea08f85740cdc8035749cce943537d7b9dbc209b244ba846"],"state_sha256":"6c9eadd48a9b7b47246f735c73cfb92b345168a1a9ad742bd950a08d1ab9b69d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vxCsEx1f/L9HpFjgPs7wrHAB5+DwMOcoLf2PCeWtmKaGvsd5ds1rq97FR7nao1+3x4dJmWCUCAPtXSlKrrpLDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T11:16:20.118753Z","bundle_sha256":"8681d4a302405401361c9aa1f101e1e772d9a8304649098da7867b58b98daba3"}}