{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RU5L6FJXEDNOK6ICG4WW5NARJK","short_pith_number":"pith:RU5L6FJX","schema_version":"1.0","canonical_sha256":"8d3abf153720dae57902372d6eb4114a80176da24ca5483da11edbedb8c38c60","source":{"kind":"arxiv","id":"2402.16827","version":3},"attestation_state":"computed","paper":{"title":"A Survey on Data Selection for Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alon Albalak, Bairu Hou, Colin Raffel, Haewon Jeong, Liangming Pan, Nathan Lambert, Niklas Muennighoff, Sang Michael Xie, Shayne Longpre, Shiyu Chang, Tatsunori Hashimoto, William Yang Wang, Xinyi Wang, Yanai Elazar","submitted_at":"2024-02-26T18:54:35Z","abstract_excerpt":"A major factor in the recent success of large language models is the use of enormous and ever-growing text datasets for unsupervised pre-training. However, naively training a model on all available data may not be optimal (or feasible), as the quality of available text data can vary. Filtering out data can also decrease the carbon footprint and financial costs of training models by reducing the amount of training required. Data selection methods aim to determine which candidate data points to include in the training dataset and how to appropriately sample from the selected data points. The pro"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.16827","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-26T18:54:35Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"b892ec169ba7c60bd533d61a043de4e3e86ffb98550ae479c460b979aea13a9e","abstract_canon_sha256":"705943456afaee5bf9df7259df2e19f7b2f9718de10551c1aae976a99a125623"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:51:37.149918Z","signature_b64":"d8NcO11buma0cCvQsRBFdGmO4As6Ihxi+MWCNdO2OkAF8Advg2jDR/DYRDY3CwUuCKETDjZ+oQV4DVIL1ZXYCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8d3abf153720dae57902372d6eb4114a80176da24ca5483da11edbedb8c38c60","last_reissued_at":"2026-07-05T08:51:37.149439Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:51:37.149439Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Survey on Data Selection for Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alon Albalak, Bairu Hou, Colin Raffel, Haewon Jeong, Liangming Pan, Nathan Lambert, Niklas Muennighoff, Sang Michael Xie, Shayne Longpre, Shiyu Chang, Tatsunori Hashimoto, William Yang Wang, Xinyi Wang, Yanai Elazar","submitted_at":"2024-02-26T18:54:35Z","abstract_excerpt":"A major factor in the recent success of large language models is the use of enormous and ever-growing text datasets for unsupervised pre-training. However, naively training a model on all available data may not be optimal (or feasible), as the quality of available text data can vary. Filtering out data can also decrease the carbon footprint and financial costs of training models by reducing the amount of training required. Data selection methods aim to determine which candidate data points to include in the training dataset and how to appropriately sample from the selected data points. The pro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.16827","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.16827/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.16827","created_at":"2026-07-05T08:51:37.149494+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.16827v3","created_at":"2026-07-05T08:51:37.149494+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.16827","created_at":"2026-07-05T08:51:37.149494+00:00"},{"alias_kind":"pith_short_12","alias_value":"RU5L6FJXEDNO","created_at":"2026-07-05T08:51:37.149494+00:00"},{"alias_kind":"pith_short_16","alias_value":"RU5L6FJXEDNOK6IC","created_at":"2026-07-05T08:51:37.149494+00:00"},{"alias_kind":"pith_short_8","alias_value":"RU5L6FJX","created_at":"2026-07-05T08:51:37.149494+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":28,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07023","citing_title":"Online Data Selection Is Implicit Alignment","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2606.26091","citing_title":"On-Policy Self-Distillation with Sampled Demonstrations Reduces Output Diversity","ref_index":215,"is_internal_anchor":false},{"citing_arxiv_id":"2606.24133","citing_title":"Holistic Data Scheduler for LLM Pre-training via Multi-Objective Reinforcement Learning","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.23611","citing_title":"Data Selection Through Iterative Self-Filtering for Vision-Language Settings","ref_index":194,"is_internal_anchor":false},{"citing_arxiv_id":"2607.02266","citing_title":"HERMES: A Multi-Granularity Labeling Substrate for Pre-training Data Mixtures","ref_index":37,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05936","citing_title":"Epistemic Injustice in Language Models: An Audit of Pretraining Filters and Guardrails","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28030","citing_title":"SPARD: Defending Harmful Fine-Tuning Attack via Safety Projection with Relevance-Diversity Data Selection","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30537","citing_title":"The Long-Term Effects of Data Selection in LLM Fine-Tuning","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.31164","citing_title":"D$^3$: Dynamic Directional Graph-Constrained Data Scheduling for LLM Training","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00251","citing_title":"Capability Self-Assessment: Teaching LLMs to Know Their Limits","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11499","citing_title":"Hubs or Fringes: Pretraining Data Selection via Web Graph Centrality","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2410.20791","citing_title":"From Cool Demos to Production-Ready FMware: Core Challenges and a Technology Roadmap","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2411.05527","citing_title":"How Good is Your Wikipedia? Auditing Data Quality for Low-resource and Multilingual NLP","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2501.01046","citing_title":"SEDD: Scalable and Efficient Dataset Deduplication with GPUs","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2502.02871","citing_title":"Position: Multimodal Large Language Models Can Significantly Advance Scientific Reasoning","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2502.00270","citing_title":"DUET: Optimizing Training Data Mixtures via Feedback from Unseen Evaluation Tasks","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22389","citing_title":"Unified Data Selection for LLM Reasoning","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2507.18406","citing_title":"Factual Inconsistencies in Multilingual Wikipedia Tables","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15691","citing_title":"SEED: Targeted Data Selection by Weighted Independent Set","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19762","citing_title":"What Really Improves Mathematical Reasoning: Structured Reasoning Signals Beyond Pure Code","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17000","citing_title":"BoLT: A Benchmark to Democratize Black-box Optimization Research for Expensive LLM Tasks","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2506.01937","citing_title":"RewardBench 2: Advancing Reward Model Evaluation","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2406.11794","citing_title":"DataComp-LM: In search of the next generation of training sets for language models","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2511.21613","citing_title":"Beyond URLs: Metadata Diversity and Position for Efficient LLM Pretraining","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2402.19173","citing_title":"StarCoder 2 and The Stack v2: The Next Generation","ref_index":153,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK","json":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK.json","graph_json":"https://pith.science/api/pith-number/RU5L6FJXEDNOK6ICG4WW5NARJK/graph.json","events_json":"https://pith.science/api/pith-number/RU5L6FJXEDNOK6ICG4WW5NARJK/events.json","paper":"https://pith.science/paper/RU5L6FJX"},"agent_actions":{"view_html":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK","download_json":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK.json","view_paper":"https://pith.science/paper/RU5L6FJX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.16827&json=true","fetch_graph":"https://pith.science/api/pith-number/RU5L6FJXEDNOK6ICG4WW5NARJK/graph.json","fetch_events":"https://pith.science/api/pith-number/RU5L6FJXEDNOK6ICG4WW5NARJK/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK/action/storage_attestation","attest_author":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK/action/author_attestation","sign_citation":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK/action/citation_signature","submit_replication":"https://pith.science/pith/RU5L6FJXEDNOK6ICG4WW5NARJK/action/replication_record"}},"created_at":"2026-07-05T08:51:37.149494+00:00","updated_at":"2026-07-05T08:51:37.149494+00:00"}