{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:O2LWNDF3WFPUSCW6BDKTV3NY5W","short_pith_number":"pith:O2LWNDF3","canonical_record":{"source":{"id":"2401.16380","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T18:19:08Z","cross_cats_sorted":[],"title_canon_sha256":"6fe0bda5f369f139201466d9aeae7c74a6b9206fc7d88a673633bb852e94a88c","abstract_canon_sha256":"fd09794eb8352bc47f1e03acf3899fadfe3331072b3d48a2b595974ba0041023"},"schema_version":"1.0"},"canonical_sha256":"7697668cbbb15f490ade08d53aedb8edaeab57dbde9799a8b5b5667868781f10","source":{"kind":"arxiv","id":"2401.16380","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16380","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16380v1","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16380","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_12","alias_value":"O2LWNDF3WFPU","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_16","alias_value":"O2LWNDF3WFPUSCW6","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_8","alias_value":"O2LWNDF3","created_at":"2026-07-05T07:38:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:O2LWNDF3WFPUSCW6BDKTV3NY5W","target":"record","payload":{"canonical_record":{"source":{"id":"2401.16380","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T18:19:08Z","cross_cats_sorted":[],"title_canon_sha256":"6fe0bda5f369f139201466d9aeae7c74a6b9206fc7d88a673633bb852e94a88c","abstract_canon_sha256":"fd09794eb8352bc47f1e03acf3899fadfe3331072b3d48a2b595974ba0041023"},"schema_version":"1.0"},"canonical_sha256":"7697668cbbb15f490ade08d53aedb8edaeab57dbde9799a8b5b5667868781f10","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:38:48.347974Z","signature_b64":"NfAIaGMPz2qGubCOb5jCezeBfHdfHfCRgbB7u2d2+oN4lHhM63Ah09lAmBu8owSOnf5zzDIjrg4wIgmhwrL0Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7697668cbbb15f490ade08d53aedb8edaeab57dbde9799a8b5b5667868781f10","last_reissued_at":"2026-07-05T07:38:48.347196Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:38:48.347196Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.16380","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i/vdccZKFaJueobZuWe1b9BuZ42cNEg+jjQFY0Sqw5bWebQyGPbVB0rYa7diGgiOFYz1DBDZ8xGLL1fpUoSdDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:26:49.201753Z"},"content_sha256":"13ff8726c13ef3a4404e22be2ee891ff114aa0382617795f91617a125b695f73","schema_version":"1.0","event_id":"sha256:13ff8726c13ef3a4404e22be2ee891ff114aa0382617795f91617a125b695f73"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:O2LWNDF3WFPUSCW6BDKTV3NY5W","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Rephrasing the Web: A Recipe for Compute and Data-Efficient Language Modeling","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"David Grangier, He Bai, Navdeep Jaitly, Pratyush Maini, Skyler Seto, Yizhe Zhang","submitted_at":"2024-01-29T18:19:08Z","abstract_excerpt":"Large language models are trained on massive scrapes of the web, which are often unstructured, noisy, and poorly phrased. Current scaling laws show that learning from such data requires an abundance of both compute and data, which grows with the size of the model being trained. This is infeasible both because of the large compute costs and duration associated with pre-training, and the impending scarcity of high-quality data on the web. In this work, we propose Web Rephrase Augmented Pre-training ($\\textbf{WRAP}$) that uses an off-the-shelf instruction-tuned model prompted to paraphrase docume"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16380","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.16380/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dn5iVzMYUcfof3J/0lXlU/STyB/aDFZxPhCMfJptLa1LAJqXpcT7ptTmdfwCqmz2HDqvb9DJwCoTFS08fvIeCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:26:49.202771Z"},"content_sha256":"2a5c4602e7ebf3d822534a28c6929fdb6717d88819d879dbce86f42d2c33b47d","schema_version":"1.0","event_id":"sha256:2a5c4602e7ebf3d822534a28c6929fdb6717d88819d879dbce86f42d2c33b47d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/bundle.json","state_url":"https://pith.science/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T22:26:49Z","links":{"resolver":"https://pith.science/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W","bundle":"https://pith.science/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/bundle.json","state":"https://pith.science/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/state.json","well_known_bundle":"https://pith.science/.well-known/pith/O2LWNDF3WFPUSCW6BDKTV3NY5W/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:O2LWNDF3WFPUSCW6BDKTV3NY5W","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fd09794eb8352bc47f1e03acf3899fadfe3331072b3d48a2b595974ba0041023","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T18:19:08Z","title_canon_sha256":"6fe0bda5f369f139201466d9aeae7c74a6b9206fc7d88a673633bb852e94a88c"},"schema_version":"1.0","source":{"id":"2401.16380","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16380","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16380v1","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16380","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_12","alias_value":"O2LWNDF3WFPU","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_16","alias_value":"O2LWNDF3WFPUSCW6","created_at":"2026-07-05T07:38:48Z"},{"alias_kind":"pith_short_8","alias_value":"O2LWNDF3","created_at":"2026-07-05T07:38:48Z"}],"graph_snapshots":[{"event_id":"sha256:2a5c4602e7ebf3d822534a28c6929fdb6717d88819d879dbce86f42d2c33b47d","target":"graph","created_at":"2026-07-05T07:38:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.16380/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models are trained on massive scrapes of the web, which are often unstructured, noisy, and poorly phrased. Current scaling laws show that learning from such data requires an abundance of both compute and data, which grows with the size of the model being trained. This is infeasible both because of the large compute costs and duration associated with pre-training, and the impending scarcity of high-quality data on the web. In this work, we propose Web Rephrase Augmented Pre-training ($\\textbf{WRAP}$) that uses an off-the-shelf instruction-tuned model prompted to paraphrase docume","authors_text":"David Grangier, He Bai, Navdeep Jaitly, Pratyush Maini, Skyler Seto, Yizhe Zhang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T18:19:08Z","title":"Rephrasing the Web: A Recipe for Compute and Data-Efficient Language Modeling"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16380","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:13ff8726c13ef3a4404e22be2ee891ff114aa0382617795f91617a125b695f73","target":"record","created_at":"2026-07-05T07:38:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fd09794eb8352bc47f1e03acf3899fadfe3331072b3d48a2b595974ba0041023","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T18:19:08Z","title_canon_sha256":"6fe0bda5f369f139201466d9aeae7c74a6b9206fc7d88a673633bb852e94a88c"},"schema_version":"1.0","source":{"id":"2401.16380","kind":"arxiv","version":1}},"canonical_sha256":"7697668cbbb15f490ade08d53aedb8edaeab57dbde9799a8b5b5667868781f10","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7697668cbbb15f490ade08d53aedb8edaeab57dbde9799a8b5b5667868781f10","first_computed_at":"2026-07-05T07:38:48.347196Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:38:48.347196Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NfAIaGMPz2qGubCOb5jCezeBfHdfHfCRgbB7u2d2+oN4lHhM63Ah09lAmBu8owSOnf5zzDIjrg4wIgmhwrL0Cw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:38:48.347974Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.16380","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:13ff8726c13ef3a4404e22be2ee891ff114aa0382617795f91617a125b695f73","sha256:2a5c4602e7ebf3d822534a28c6929fdb6717d88819d879dbce86f42d2c33b47d"],"state_sha256":"d8471a7557a6d743c8906af356f7b47a59728341c3ff0d4a178b1ba1f074c3d4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CR58ZIiBPE+E7YuLk7PQYe9gaiTD02mTt31PsrFCa0VPqlA99uECi7QFoQ+sKJ4WJoJ+VbAWo0sv0cqbOSZOBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T22:26:49.208526Z","bundle_sha256":"56c9f1fbcade6980b676af22ba66e3f524ffaf4dc9fe290321f2d26d5b5315e6"}}