{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BAA5CPNQ6IATJXJYGPXOC33L2T","short_pith_number":"pith:BAA5CPNQ","canonical_record":{"source":{"id":"2404.03626","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:48:28Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"856b69e14bb6d1bc77d9fd6a16be1b30089946c05ce816be93ef693b0be76851","abstract_canon_sha256":"1ae0bc1d65fc8999cdab2a19c82751e10ccb3431a33325537c06cf4156a59b6d"},"schema_version":"1.0"},"canonical_sha256":"0801d13db0f20134dd3833eee16f6bd4d7c88db72f8b3ae7903d5923d584370f","source":{"kind":"arxiv","id":"2404.03626","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.03626","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"arxiv_version","alias_value":"2404.03626v3","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.03626","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_12","alias_value":"BAA5CPNQ6IAT","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_16","alias_value":"BAA5CPNQ6IATJXJY","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_8","alias_value":"BAA5CPNQ","created_at":"2026-07-05T09:48:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BAA5CPNQ6IATJXJYGPXOC33L2T","target":"record","payload":{"canonical_record":{"source":{"id":"2404.03626","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:48:28Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"856b69e14bb6d1bc77d9fd6a16be1b30089946c05ce816be93ef693b0be76851","abstract_canon_sha256":"1ae0bc1d65fc8999cdab2a19c82751e10ccb3431a33325537c06cf4156a59b6d"},"schema_version":"1.0"},"canonical_sha256":"0801d13db0f20134dd3833eee16f6bd4d7c88db72f8b3ae7903d5923d584370f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:48:32.265712Z","signature_b64":"xBb6pKMlT1csMycNCctuUUMGaFVa9JTvX2iwrAhrO9NYrnwTeeC3ImlYVWCDguX97runSvOGa5HWmrNQVkHyBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0801d13db0f20134dd3833eee16f6bd4d7c88db72f8b3ae7903d5923d584370f","last_reissued_at":"2026-07-05T09:48:32.265217Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:48:32.265217Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.03626","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:48:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fUpV1eSKRNwmdN7VjPsHvrg+c5WUW8NnZSli3Xn9zCRwRncKyKFiUZN501oeoQQFLLHY2yb3YEvkrcqyo1FNBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T15:54:36.443746Z"},"content_sha256":"4457778ae94e15898702015572154388a99d9c4b91258e95fc7b1d23ffba035e","schema_version":"1.0","event_id":"sha256:4457778ae94e15898702015572154388a99d9c4b91258e95fc7b1d23ffba035e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BAA5CPNQ6IATJXJYGPXOC33L2T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Training LLMs over Neurally Compressed Text","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Adam Roberts, Alex Alemi, Brian Lester, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, Noah Constant","submitted_at":"2024-04-04T17:48:28Z","abstract_excerpt":"In this paper, we explore the idea of training large language models (LLMs) over highly compressed text. While standard subword tokenizers compress text by a small factor, neural text compressors can achieve much higher rates of compression. If it were possible to train LLMs directly over neurally compressed text, this would confer advantages in training and serving efficiency, as well as easier handling of long text spans. The main obstacle to this goal is that strong compression tends to produce opaque outputs that are not well-suited for learning. In particular, we find that text na\\\"ively "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.03626","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.03626/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:48:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2hZlGrF5KteW61I2Y9q8JAS2SkvTxbKjEvj7yf1U/1h3XrmFqvty8PLBCa/NBErA4v9gi0z3IKkodk8II7ZbDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T15:54:36.444247Z"},"content_sha256":"12005ab366656c646c6c994fee09c99df65ee3e0797c04214179497ffcd0024a","schema_version":"1.0","event_id":"sha256:12005ab366656c646c6c994fee09c99df65ee3e0797c04214179497ffcd0024a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/bundle.json","state_url":"https://pith.science/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T15:54:36Z","links":{"resolver":"https://pith.science/pith/BAA5CPNQ6IATJXJYGPXOC33L2T","bundle":"https://pith.science/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/bundle.json","state":"https://pith.science/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BAA5CPNQ6IATJXJYGPXOC33L2T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BAA5CPNQ6IATJXJYGPXOC33L2T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1ae0bc1d65fc8999cdab2a19c82751e10ccb3431a33325537c06cf4156a59b6d","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:48:28Z","title_canon_sha256":"856b69e14bb6d1bc77d9fd6a16be1b30089946c05ce816be93ef693b0be76851"},"schema_version":"1.0","source":{"id":"2404.03626","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.03626","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"arxiv_version","alias_value":"2404.03626v3","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.03626","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_12","alias_value":"BAA5CPNQ6IAT","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_16","alias_value":"BAA5CPNQ6IATJXJY","created_at":"2026-07-05T09:48:32Z"},{"alias_kind":"pith_short_8","alias_value":"BAA5CPNQ","created_at":"2026-07-05T09:48:32Z"}],"graph_snapshots":[{"event_id":"sha256:12005ab366656c646c6c994fee09c99df65ee3e0797c04214179497ffcd0024a","target":"graph","created_at":"2026-07-05T09:48:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.03626/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we explore the idea of training large language models (LLMs) over highly compressed text. While standard subword tokenizers compress text by a small factor, neural text compressors can achieve much higher rates of compression. If it were possible to train LLMs directly over neurally compressed text, this would confer advantages in training and serving efficiency, as well as easier handling of long text spans. The main obstacle to this goal is that strong compression tends to produce opaque outputs that are not well-suited for learning. In particular, we find that text na\\\"ively ","authors_text":"Adam Roberts, Alex Alemi, Brian Lester, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, Noah Constant","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:48:28Z","title":"Training LLMs over Neurally Compressed Text"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.03626","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4457778ae94e15898702015572154388a99d9c4b91258e95fc7b1d23ffba035e","target":"record","created_at":"2026-07-05T09:48:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1ae0bc1d65fc8999cdab2a19c82751e10ccb3431a33325537c06cf4156a59b6d","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:48:28Z","title_canon_sha256":"856b69e14bb6d1bc77d9fd6a16be1b30089946c05ce816be93ef693b0be76851"},"schema_version":"1.0","source":{"id":"2404.03626","kind":"arxiv","version":3}},"canonical_sha256":"0801d13db0f20134dd3833eee16f6bd4d7c88db72f8b3ae7903d5923d584370f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0801d13db0f20134dd3833eee16f6bd4d7c88db72f8b3ae7903d5923d584370f","first_computed_at":"2026-07-05T09:48:32.265217Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:48:32.265217Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xBb6pKMlT1csMycNCctuUUMGaFVa9JTvX2iwrAhrO9NYrnwTeeC3ImlYVWCDguX97runSvOGa5HWmrNQVkHyBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:48:32.265712Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.03626","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4457778ae94e15898702015572154388a99d9c4b91258e95fc7b1d23ffba035e","sha256:12005ab366656c646c6c994fee09c99df65ee3e0797c04214179497ffcd0024a"],"state_sha256":"d89071fbd185d4dfc71e213d345b6a5840f170eb99cb02451a2b760561846e5e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"j6Uv1N3TqQys8RwBfFhxxCuxpxIIU77jYNj3JcszrKf0R4RGukug3oNsZI8uFeNUsnSq1czwdDvOD18ZDS5UAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T15:54:36.460276Z","bundle_sha256":"8e5334774b98cba831b19731038aa56ab0352509fbf807f6b5bb0165108cbd0c"}}