{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:7MH45DVJ3Y3DXOCAUTOCVUL7RU","short_pith_number":"pith:7MH45DVJ","canonical_record":{"source":{"id":"2308.12284","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"f0c15fe2c6724d60c0571ab37e1c7bd9102bc396aa6565aab16d578c7601c5f7","abstract_canon_sha256":"3386faf85c32f1422cfa20342c97bbca7d9b819ac0f89547ac56e855ddd9b801"},"schema_version":"1.0"},"canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","source":{"kind":"arxiv","id":"2308.12284","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.12284","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"arxiv_version","alias_value":"2308.12284v1","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.12284","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_12","alias_value":"7MH45DVJ3Y3D","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_16","alias_value":"7MH45DVJ3Y3DXOCA","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_8","alias_value":"7MH45DVJ","created_at":"2026-07-05T06:44:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:7MH45DVJ3Y3DXOCAUTOCVUL7RU","target":"record","payload":{"canonical_record":{"source":{"id":"2308.12284","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"f0c15fe2c6724d60c0571ab37e1c7bd9102bc396aa6565aab16d578c7601c5f7","abstract_canon_sha256":"3386faf85c32f1422cfa20342c97bbca7d9b819ac0f89547ac56e855ddd9b801"},"schema_version":"1.0"},"canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:44:02.656010Z","signature_b64":"0EgNX5S05I/qekUPmovNmRmnHtds3W9O6shteD3C3IscXR2sfAVJ1gl9NHZspPKe6E82D+CorTGMEYA1gHljDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","last_reissued_at":"2026-07-05T06:44:02.655508Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:44:02.655508Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.12284","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:44:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XmC6HCkDIRYdAPTjSgIa0p2abzD7PzkEZUCqhhYKxSuEszI89BDA971VQ3WWjuPyR3AwvEvBVCIBt0PM3oANDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T17:24:58.725509Z"},"content_sha256":"53e96b5e4f3a7368f291e9ef4ca02a3387dd2dd7124e8d7325e1334edfa6aba5","schema_version":"1.0","event_id":"sha256:53e96b5e4f3a7368f291e9ef4ca02a3387dd2dd7124e8d7325e1334edfa6aba5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:7MH45DVJ3Y3DXOCAUTOCVUL7RU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"D4: Improving LLM Pretraining via Document De-Duplication and Diversification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ari S. Morcos, Armen Aghajanyan, Daniel Simig, Kushal Tirumala","submitted_at":"2023-08-23T17:58:14Z","abstract_excerpt":"Over recent years, an increasing amount of compute and data has been poured into training large language models (LLMs), usually by doing one-pass learning on as many tokens as possible randomly selected from large-scale web corpora. While training on ever-larger portions of the internet leads to consistent performance improvements, the size of these improvements diminishes with scale, and there has been little work exploring the effect of data selection on pre-training and downstream performance beyond simple de-duplication methods such as MinHash. Here, we show that careful data selection (on"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.12284","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.12284/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:44:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dmX7CmVHyrpBheYkPmru3EksdVpRlf0AahVvP8DIOTHR9wqpT3Akq6QFGi/57YiZs+YoDDQClJ4zV335XjMnAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T17:24:58.725957Z"},"content_sha256":"e825a71491abb8dea6016872df5a26e92bc75efb88b06dba2cf6563e07d31c59","schema_version":"1.0","event_id":"sha256:e825a71491abb8dea6016872df5a26e92bc75efb88b06dba2cf6563e07d31c59"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/bundle.json","state_url":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T17:24:58Z","links":{"resolver":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU","bundle":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/bundle.json","state":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:7MH45DVJ3Y3DXOCAUTOCVUL7RU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3386faf85c32f1422cfa20342c97bbca7d9b819ac0f89547ac56e855ddd9b801","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","title_canon_sha256":"f0c15fe2c6724d60c0571ab37e1c7bd9102bc396aa6565aab16d578c7601c5f7"},"schema_version":"1.0","source":{"id":"2308.12284","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.12284","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"arxiv_version","alias_value":"2308.12284v1","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.12284","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_12","alias_value":"7MH45DVJ3Y3D","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_16","alias_value":"7MH45DVJ3Y3DXOCA","created_at":"2026-07-05T06:44:02Z"},{"alias_kind":"pith_short_8","alias_value":"7MH45DVJ","created_at":"2026-07-05T06:44:02Z"}],"graph_snapshots":[{"event_id":"sha256:e825a71491abb8dea6016872df5a26e92bc75efb88b06dba2cf6563e07d31c59","target":"graph","created_at":"2026-07-05T06:44:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.12284/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Over recent years, an increasing amount of compute and data has been poured into training large language models (LLMs), usually by doing one-pass learning on as many tokens as possible randomly selected from large-scale web corpora. While training on ever-larger portions of the internet leads to consistent performance improvements, the size of these improvements diminishes with scale, and there has been little work exploring the effect of data selection on pre-training and downstream performance beyond simple de-duplication methods such as MinHash. Here, we show that careful data selection (on","authors_text":"Ari S. Morcos, Armen Aghajanyan, Daniel Simig, Kushal Tirumala","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","title":"D4: Improving LLM Pretraining via Document De-Duplication and Diversification"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.12284","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53e96b5e4f3a7368f291e9ef4ca02a3387dd2dd7124e8d7325e1334edfa6aba5","target":"record","created_at":"2026-07-05T06:44:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3386faf85c32f1422cfa20342c97bbca7d9b819ac0f89547ac56e855ddd9b801","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","title_canon_sha256":"f0c15fe2c6724d60c0571ab37e1c7bd9102bc396aa6565aab16d578c7601c5f7"},"schema_version":"1.0","source":{"id":"2308.12284","kind":"arxiv","version":1}},"canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","first_computed_at":"2026-07-05T06:44:02.655508Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:44:02.655508Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0EgNX5S05I/qekUPmovNmRmnHtds3W9O6shteD3C3IscXR2sfAVJ1gl9NHZspPKe6E82D+CorTGMEYA1gHljDg==","signature_status":"signed_v1","signed_at":"2026-07-05T06:44:02.656010Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.12284","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53e96b5e4f3a7368f291e9ef4ca02a3387dd2dd7124e8d7325e1334edfa6aba5","sha256:e825a71491abb8dea6016872df5a26e92bc75efb88b06dba2cf6563e07d31c59"],"state_sha256":"2016e003a1b962fe2837fdcbb1c29454595dc54e2cb0e8aa0f09e898d22460c4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LKOJPKKOMFENXFxavsimm6FCumAnkP62q9qDFWi2DBnCwOLJDCLlsypNS9MgorC01ZSX1dQXASQPujjQTA4nDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T17:24:58.727873Z","bundle_sha256":"1658e95e14b509424e64704dcf3737ce2a203e70b12abe4876410227b39104b6"}}