{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:PLWMQ7BSLD2G4IDO7IRPH4VGQH","short_pith_number":"pith:PLWMQ7BS","canonical_record":{"source":{"id":"2502.06604","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T16:01:55Z","cross_cats_sorted":[],"title_canon_sha256":"2a3365c4cebcc780605bba3810732ea9d798128cdb160675afc7ade6a277dabe","abstract_canon_sha256":"93c1dccf7d8c2c310327468510a1b60b61bfa1ccf1e1d1151f8b043f4624cf8b"},"schema_version":"1.0"},"canonical_sha256":"7aecc87c3258f46e206efa22f3f2a681c0da5b54cc8074877c87d3795ec70f17","source":{"kind":"arxiv","id":"2502.06604","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.06604","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"arxiv_version","alias_value":"2502.06604v2","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.06604","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_12","alias_value":"PLWMQ7BSLD2G","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_16","alias_value":"PLWMQ7BSLD2G4IDO","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_8","alias_value":"PLWMQ7BS","created_at":"2026-07-05T11:03:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:PLWMQ7BSLD2G4IDO7IRPH4VGQH","target":"record","payload":{"canonical_record":{"source":{"id":"2502.06604","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T16:01:55Z","cross_cats_sorted":[],"title_canon_sha256":"2a3365c4cebcc780605bba3810732ea9d798128cdb160675afc7ade6a277dabe","abstract_canon_sha256":"93c1dccf7d8c2c310327468510a1b60b61bfa1ccf1e1d1151f8b043f4624cf8b"},"schema_version":"1.0"},"canonical_sha256":"7aecc87c3258f46e206efa22f3f2a681c0da5b54cc8074877c87d3795ec70f17","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:03:56.447613Z","signature_b64":"C4eH5Yh034AXaMNPDvaFGxYEQiiiQ++KA5muFFYrkSoTb+wnI/Hs1szHoioH2UcLMUbGyvDDx6/aPsyUa/TNAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7aecc87c3258f46e206efa22f3f2a681c0da5b54cc8074877c87d3795ec70f17","last_reissued_at":"2026-07-05T11:03:56.447150Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:03:56.447150Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.06604","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0Jw3vgUWHp8zn4S5Gf+ONHGBCKETz5GykDe9VqUfX8p7ZXkxJQHGTXFm3wx05+ed8GXXKhw6dnmmUxPWh35uCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-23T12:21:48.455907Z"},"content_sha256":"179f31cb63612f8fb097656ab624d0e3b90e437203422222a9eb490b58fabb66","schema_version":"1.0","event_id":"sha256:179f31cb63612f8fb097656ab624d0e3b90e437203422222a9eb490b58fabb66"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:PLWMQ7BSLD2G4IDO7IRPH4VGQH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Do we really have to filter out random noise in pre-training data for language models?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jinghan Ru, Qianli Ren, Xianwei Zhuang, Yuexian Zou, Yuguo Yin, Yuxin Xie, Zhihui Guo, Zhiming Liu","submitted_at":"2025-02-10T16:01:55Z","abstract_excerpt":"Web-scale pre-training datasets are the cornerstone of LLMs' success. However, text data curated from the Internet inevitably contains random noise caused by decoding errors or unregulated web content. In contrast to previous works that focus on low quality or synthetic data, our study \\textbf{provides the first systematic investigation of such random noise through a cohesive ``What-Why-How'' framework.} Surprisingly, we observed that the resulting increase in the loss of next-token prediction (NTP) was significantly lower than the proportion of random noise even when the model was scaled up t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.06604","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.06604/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XfH0tasO9F4mu5DSSBDmWmQ8gJapyz4EXx/DY8k5PaV1/qyJzMuLjryJNsXXDi76XX73Sq1KcwGnIprcpwXNDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-23T12:21:48.456279Z"},"content_sha256":"f5ea17579f50103cd207afb840ec0f92113bdefb5a922ff8fd0ab31b8d33ea9d","schema_version":"1.0","event_id":"sha256:f5ea17579f50103cd207afb840ec0f92113bdefb5a922ff8fd0ab31b8d33ea9d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/bundle.json","state_url":"https://pith.science/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-23T12:21:48Z","links":{"resolver":"https://pith.science/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH","bundle":"https://pith.science/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/bundle.json","state":"https://pith.science/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PLWMQ7BSLD2G4IDO7IRPH4VGQH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:PLWMQ7BSLD2G4IDO7IRPH4VGQH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"93c1dccf7d8c2c310327468510a1b60b61bfa1ccf1e1d1151f8b043f4624cf8b","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T16:01:55Z","title_canon_sha256":"2a3365c4cebcc780605bba3810732ea9d798128cdb160675afc7ade6a277dabe"},"schema_version":"1.0","source":{"id":"2502.06604","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.06604","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"arxiv_version","alias_value":"2502.06604v2","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.06604","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_12","alias_value":"PLWMQ7BSLD2G","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_16","alias_value":"PLWMQ7BSLD2G4IDO","created_at":"2026-07-05T11:03:56Z"},{"alias_kind":"pith_short_8","alias_value":"PLWMQ7BS","created_at":"2026-07-05T11:03:56Z"}],"graph_snapshots":[{"event_id":"sha256:f5ea17579f50103cd207afb840ec0f92113bdefb5a922ff8fd0ab31b8d33ea9d","target":"graph","created_at":"2026-07-05T11:03:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.06604/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Web-scale pre-training datasets are the cornerstone of LLMs' success. However, text data curated from the Internet inevitably contains random noise caused by decoding errors or unregulated web content. In contrast to previous works that focus on low quality or synthetic data, our study \\textbf{provides the first systematic investigation of such random noise through a cohesive ``What-Why-How'' framework.} Surprisingly, we observed that the resulting increase in the loss of next-token prediction (NTP) was significantly lower than the proportion of random noise even when the model was scaled up t","authors_text":"Jinghan Ru, Qianli Ren, Xianwei Zhuang, Yuexian Zou, Yuguo Yin, Yuxin Xie, Zhihui Guo, Zhiming Liu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T16:01:55Z","title":"Do we really have to filter out random noise in pre-training data for language models?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.06604","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:179f31cb63612f8fb097656ab624d0e3b90e437203422222a9eb490b58fabb66","target":"record","created_at":"2026-07-05T11:03:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"93c1dccf7d8c2c310327468510a1b60b61bfa1ccf1e1d1151f8b043f4624cf8b","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T16:01:55Z","title_canon_sha256":"2a3365c4cebcc780605bba3810732ea9d798128cdb160675afc7ade6a277dabe"},"schema_version":"1.0","source":{"id":"2502.06604","kind":"arxiv","version":2}},"canonical_sha256":"7aecc87c3258f46e206efa22f3f2a681c0da5b54cc8074877c87d3795ec70f17","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7aecc87c3258f46e206efa22f3f2a681c0da5b54cc8074877c87d3795ec70f17","first_computed_at":"2026-07-05T11:03:56.447150Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:03:56.447150Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"C4eH5Yh034AXaMNPDvaFGxYEQiiiQ++KA5muFFYrkSoTb+wnI/Hs1szHoioH2UcLMUbGyvDDx6/aPsyUa/TNAw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:03:56.447613Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.06604","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:179f31cb63612f8fb097656ab624d0e3b90e437203422222a9eb490b58fabb66","sha256:f5ea17579f50103cd207afb840ec0f92113bdefb5a922ff8fd0ab31b8d33ea9d"],"state_sha256":"ce3b209c77fba81c1ad08e68bc4914dc13f57814e3cf73301fe908f5cf7f676b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BUFOfzP4U9EBRuG4n8Q3jJdzH3/pfST3JaYrXH3IOuGwpSKb4VQttNDlGNoiljecfYQKmpRIim1PHANBlJWNBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-23T12:21:48.458836Z","bundle_sha256":"0a36db5907e81dd338bd960143d81fdfea084f7922a0f234b03476032193056d"}}