{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:7MH45DVJ3Y3DXOCAUTOCVUL7RU","short_pith_number":"pith:7MH45DVJ","schema_version":"1.0","canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","source":{"kind":"arxiv","id":"2308.12284","version":1},"attestation_state":"computed","paper":{"title":"D4: Improving LLM Pretraining via Document De-Duplication and Diversification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ari S. Morcos, Armen Aghajanyan, Daniel Simig, Kushal Tirumala","submitted_at":"2023-08-23T17:58:14Z","abstract_excerpt":"Over recent years, an increasing amount of compute and data has been poured into training large language models (LLMs), usually by doing one-pass learning on as many tokens as possible randomly selected from large-scale web corpora. While training on ever-larger portions of the internet leads to consistent performance improvements, the size of these improvements diminishes with scale, and there has been little work exploring the effect of data selection on pre-training and downstream performance beyond simple de-duplication methods such as MinHash. Here, we show that careful data selection (on"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2308.12284","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-23T17:58:14Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"f0c15fe2c6724d60c0571ab37e1c7bd9102bc396aa6565aab16d578c7601c5f7","abstract_canon_sha256":"3386faf85c32f1422cfa20342c97bbca7d9b819ac0f89547ac56e855ddd9b801"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:44:02.656010Z","signature_b64":"0EgNX5S05I/qekUPmovNmRmnHtds3W9O6shteD3C3IscXR2sfAVJ1gl9NHZspPKe6E82D+CorTGMEYA1gHljDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fb0fce8ea9de363bb840a4dc2ad17f8d3212b6ccc2fc2c3015f93d0dded681f9","last_reissued_at":"2026-07-05T06:44:02.655508Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:44:02.655508Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"D4: Improving LLM Pretraining via Document De-Duplication and Diversification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ari S. Morcos, Armen Aghajanyan, Daniel Simig, Kushal Tirumala","submitted_at":"2023-08-23T17:58:14Z","abstract_excerpt":"Over recent years, an increasing amount of compute and data has been poured into training large language models (LLMs), usually by doing one-pass learning on as many tokens as possible randomly selected from large-scale web corpora. While training on ever-larger portions of the internet leads to consistent performance improvements, the size of these improvements diminishes with scale, and there has been little work exploring the effect of data selection on pre-training and downstream performance beyond simple de-duplication methods such as MinHash. Here, we show that careful data selection (on"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.12284","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.12284/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2308.12284","created_at":"2026-07-05T06:44:02.655570+00:00"},{"alias_kind":"arxiv_version","alias_value":"2308.12284v1","created_at":"2026-07-05T06:44:02.655570+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.12284","created_at":"2026-07-05T06:44:02.655570+00:00"},{"alias_kind":"pith_short_12","alias_value":"7MH45DVJ3Y3D","created_at":"2026-07-05T06:44:02.655570+00:00"},{"alias_kind":"pith_short_16","alias_value":"7MH45DVJ3Y3DXOCA","created_at":"2026-07-05T06:44:02.655570+00:00"},{"alias_kind":"pith_short_8","alias_value":"7MH45DVJ","created_at":"2026-07-05T06:44:02.655570+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2411.05527","citing_title":"How Good is Your Wikipedia? Auditing Data Quality for Low-resource and Multilingual NLP","ref_index":57,"is_internal_anchor":false},{"citing_arxiv_id":"2509.07177","citing_title":"Towards EnergyGPT: A Large Language Model Specialized for the Energy Sector","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2309.16671","citing_title":"Demystifying CLIP Data","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09990","citing_title":"Merlin: Deterministic Byte-Exact Deduplication for Lossless Context Optimization in Large Language Model Inference","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09611","citing_title":"Byte-Exact Deduplication in Retrieval-Augmented Generation: A Three-Regime Empirical Analysis Across Public Benchmarks","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2303.18223","citing_title":"A Survey of Large Language Models","ref_index":252,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16197","citing_title":"Sketching the Readout of Large Language Models for Scalable Data Attribution and Valuation","ref_index":50,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU","json":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU.json","graph_json":"https://pith.science/api/pith-number/7MH45DVJ3Y3DXOCAUTOCVUL7RU/graph.json","events_json":"https://pith.science/api/pith-number/7MH45DVJ3Y3DXOCAUTOCVUL7RU/events.json","paper":"https://pith.science/paper/7MH45DVJ"},"agent_actions":{"view_html":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU","download_json":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU.json","view_paper":"https://pith.science/paper/7MH45DVJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2308.12284&json=true","fetch_graph":"https://pith.science/api/pith-number/7MH45DVJ3Y3DXOCAUTOCVUL7RU/graph.json","fetch_events":"https://pith.science/api/pith-number/7MH45DVJ3Y3DXOCAUTOCVUL7RU/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/action/storage_attestation","attest_author":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/action/author_attestation","sign_citation":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/action/citation_signature","submit_replication":"https://pith.science/pith/7MH45DVJ3Y3DXOCAUTOCVUL7RU/action/replication_record"}},"created_at":"2026-07-05T06:44:02.655570+00:00","updated_at":"2026-07-05T06:44:02.655570+00:00"}