{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:OF44L4LN6B35I7SRNJ3OAVFKG3","short_pith_number":"pith:OF44L4LN","canonical_record":{"source":{"id":"2305.13169","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T15:57:53Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"9b98041546963af5c71d879c1f0709bfcd4e6aeb6343e163be2e2de57acd83bc","abstract_canon_sha256":"3702ad2f24882dc1889761569c6bee0b2c31b21b1e1470cd263ee654ef936371"},"schema_version":"1.0"},"canonical_sha256":"7179c5f16df077d47e516a76e054aa36f1e504ce948585e43a83d599528f17fe","source":{"kind":"arxiv","id":"2305.13169","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13169","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13169v2","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13169","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_12","alias_value":"OF44L4LN6B35","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_16","alias_value":"OF44L4LN6B35I7SR","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_8","alias_value":"OF44L4LN","created_at":"2026-07-05T07:11:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:OF44L4LN6B35I7SRNJ3OAVFKG3","target":"record","payload":{"canonical_record":{"source":{"id":"2305.13169","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T15:57:53Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"9b98041546963af5c71d879c1f0709bfcd4e6aeb6343e163be2e2de57acd83bc","abstract_canon_sha256":"3702ad2f24882dc1889761569c6bee0b2c31b21b1e1470cd263ee654ef936371"},"schema_version":"1.0"},"canonical_sha256":"7179c5f16df077d47e516a76e054aa36f1e504ce948585e43a83d599528f17fe","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:11:55.968487Z","signature_b64":"yfelYIIn95IqQxu1WnKUXxtTtXNpYWVZiX17y76Tq5wP64XPg1pxokjf2yVANyR3LsHiJBLxbeiOUUbP5igIAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7179c5f16df077d47e516a76e054aa36f1e504ce948585e43a83d599528f17fe","last_reissued_at":"2026-07-05T07:11:55.968001Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:11:55.968001Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.13169","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:11:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jrwi4lvF4nb94anFG1Ep5D5CB8xoslsTfw43r2oqBLnBt6XnHsPr7MtjWhVzXo7P61lQUHjRZH3O0E9uzWrJCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:19:07.305906Z"},"content_sha256":"7390c84b8855df1930e7878113cb2d7375299ff6a8ed708c556c78b7fd5516a2","schema_version":"1.0","event_id":"sha256:7390c84b8855df1930e7878113cb2d7375299ff6a8ed708c556c78b7fd5516a2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:OF44L4LN6B35I7SRNJ3OAVFKG3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Pretrainer's Guide to Training Data: Measuring the Effects of Data Age, Domain Coverage, Quality, & Toxicity","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Adam Roberts, Barret Zoph, Daphne Ippolito, David Mimno, Denny Zhou, Emily Reif, Gregory Yauney, Jason Wei, Katherine Lee, Kevin Robinson, Shayne Longpre","submitted_at":"2023-05-22T15:57:53Z","abstract_excerpt":"Pretraining is the preliminary and fundamental step in developing capable language models (LM). Despite this, pretraining data design is critically under-documented and often guided by empirically unsupported intuitions. To address this, we pretrain 28 1.5B parameter decoder-only models, training on data curated (1) at different times, (2) with varying toxicity and quality filters, and (3) with different domain compositions. First, we quantify the effect of pretraining data age. A temporal shift between evaluation data and pretraining data leads to performance degradation, which is not overcom"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13169","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.13169/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:11:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3NTQFHLKnpi8/idyGLF2RB6udPgjAQuBXVuvd7yFO/2c/FFnb9wcvo6JtPD8w59zx+uoNMvgOw9GEJR1bJ9LCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:19:07.306410Z"},"content_sha256":"7155246012daee29f4a3abe9d01f9983997b4ae3d6890ead70b291633d5ff88b","schema_version":"1.0","event_id":"sha256:7155246012daee29f4a3abe9d01f9983997b4ae3d6890ead70b291633d5ff88b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/bundle.json","state_url":"https://pith.science/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T17:19:07Z","links":{"resolver":"https://pith.science/pith/OF44L4LN6B35I7SRNJ3OAVFKG3","bundle":"https://pith.science/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/bundle.json","state":"https://pith.science/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OF44L4LN6B35I7SRNJ3OAVFKG3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:OF44L4LN6B35I7SRNJ3OAVFKG3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3702ad2f24882dc1889761569c6bee0b2c31b21b1e1470cd263ee654ef936371","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T15:57:53Z","title_canon_sha256":"9b98041546963af5c71d879c1f0709bfcd4e6aeb6343e163be2e2de57acd83bc"},"schema_version":"1.0","source":{"id":"2305.13169","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13169","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13169v2","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13169","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_12","alias_value":"OF44L4LN6B35","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_16","alias_value":"OF44L4LN6B35I7SR","created_at":"2026-07-05T07:11:55Z"},{"alias_kind":"pith_short_8","alias_value":"OF44L4LN","created_at":"2026-07-05T07:11:55Z"}],"graph_snapshots":[{"event_id":"sha256:7155246012daee29f4a3abe9d01f9983997b4ae3d6890ead70b291633d5ff88b","target":"graph","created_at":"2026-07-05T07:11:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.13169/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Pretraining is the preliminary and fundamental step in developing capable language models (LM). Despite this, pretraining data design is critically under-documented and often guided by empirically unsupported intuitions. To address this, we pretrain 28 1.5B parameter decoder-only models, training on data curated (1) at different times, (2) with varying toxicity and quality filters, and (3) with different domain compositions. First, we quantify the effect of pretraining data age. A temporal shift between evaluation data and pretraining data leads to performance degradation, which is not overcom","authors_text":"Adam Roberts, Barret Zoph, Daphne Ippolito, David Mimno, Denny Zhou, Emily Reif, Gregory Yauney, Jason Wei, Katherine Lee, Kevin Robinson, Shayne Longpre","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T15:57:53Z","title":"A Pretrainer's Guide to Training Data: Measuring the Effects of Data Age, Domain Coverage, Quality, & Toxicity"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13169","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7390c84b8855df1930e7878113cb2d7375299ff6a8ed708c556c78b7fd5516a2","target":"record","created_at":"2026-07-05T07:11:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3702ad2f24882dc1889761569c6bee0b2c31b21b1e1470cd263ee654ef936371","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T15:57:53Z","title_canon_sha256":"9b98041546963af5c71d879c1f0709bfcd4e6aeb6343e163be2e2de57acd83bc"},"schema_version":"1.0","source":{"id":"2305.13169","kind":"arxiv","version":2}},"canonical_sha256":"7179c5f16df077d47e516a76e054aa36f1e504ce948585e43a83d599528f17fe","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7179c5f16df077d47e516a76e054aa36f1e504ce948585e43a83d599528f17fe","first_computed_at":"2026-07-05T07:11:55.968001Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:11:55.968001Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yfelYIIn95IqQxu1WnKUXxtTtXNpYWVZiX17y76Tq5wP64XPg1pxokjf2yVANyR3LsHiJBLxbeiOUUbP5igIAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T07:11:55.968487Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.13169","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7390c84b8855df1930e7878113cb2d7375299ff6a8ed708c556c78b7fd5516a2","sha256:7155246012daee29f4a3abe9d01f9983997b4ae3d6890ead70b291633d5ff88b"],"state_sha256":"5159b2abac31a732d0fcc3a2be16aaf915665679d25ab687bb718f1a0bdc6739"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"guYB+ne81ZZcX45NF4bd1A5u0+e3LznrLbDvYkBrlw9vpViwXI/T6DmLlYrwORi6lMtX6o1umoR9OEmiReqsAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T17:19:07.311880Z","bundle_sha256":"d5d4456c687a6474679818d17bf2dd67a5242abdd2c2ccd2e69ad798ac1af813"}}