{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:IAJI7FIYDI3AQEMITO7AUPW7DG","short_pith_number":"pith:IAJI7FIY","canonical_record":{"source":{"id":"2407.03678","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T06:52:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"60e020aa2bbd4221e5b4ed0bb2b1363fba3dfa0bd20682637815eb3ef95f4db7","abstract_canon_sha256":"fda304c5ad44b0b628a4771a874940186e399b0fae6a44bbcfcf51dab2aa304b"},"schema_version":"1.0"},"canonical_sha256":"40128f95181a360811889bbe0a3edf19aa4d8fe14c4fc4b3d1a82c76f1af114e","source":{"kind":"arxiv","id":"2407.03678","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.03678","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"arxiv_version","alias_value":"2407.03678v1","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.03678","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_12","alias_value":"IAJI7FIYDI3A","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_16","alias_value":"IAJI7FIYDI3AQEMI","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_8","alias_value":"IAJI7FIY","created_at":"2026-07-05T08:40:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:IAJI7FIYDI3AQEMITO7AUPW7DG","target":"record","payload":{"canonical_record":{"source":{"id":"2407.03678","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T06:52:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"60e020aa2bbd4221e5b4ed0bb2b1363fba3dfa0bd20682637815eb3ef95f4db7","abstract_canon_sha256":"fda304c5ad44b0b628a4771a874940186e399b0fae6a44bbcfcf51dab2aa304b"},"schema_version":"1.0"},"canonical_sha256":"40128f95181a360811889bbe0a3edf19aa4d8fe14c4fc4b3d1a82c76f1af114e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:40:21.928787Z","signature_b64":"X76hvtU6sw0V7PO4PF82RDwMKz3zZ9pPjQLm/35JxTHzCMouevcx02n5ROa/39clToxUQb8WDlbAiehe7EGNDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"40128f95181a360811889bbe0a3edf19aa4d8fe14c4fc4b3d1a82c76f1af114e","last_reissued_at":"2026-07-05T08:40:21.928292Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:40:21.928292Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.03678","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MhKlbSaGhacDqjcgoU9k950fRU8KJGTfBBGohFzmGLtFgOSOu0zuoGehMDHSpBmVGXgXjCBn2kB8k+rnxbISDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T01:30:50.962230Z"},"content_sha256":"94c7d46d5164aa22d66b77e5d4c788d0363689701e6d46f2b0bfd7635355529a","schema_version":"1.0","event_id":"sha256:94c7d46d5164aa22d66b77e5d4c788d0363689701e6d46f2b0bfd7635355529a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:IAJI7FIYDI3AQEMITO7AUPW7DG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Improving Self Consistency in LLMs through Probabilistic Tokenization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ashutosh Sathe, Divyanshu Aggarwal, Sunayana Sitaram","submitted_at":"2024-07-04T06:52:48Z","abstract_excerpt":"Prior research has demonstrated noticeable performance gains through the use of probabilistic tokenizations, an approach that involves employing multiple tokenizations of the same input string during the training phase of a language model. Despite these promising findings, modern large language models (LLMs) have yet to be trained using probabilistic tokenizations. Interestingly, while the tokenizers of these contemporary LLMs have the capability to generate multiple tokenizations, this property remains underutilized.\n  In this work, we propose a novel method to leverage the multiple tokenizat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.03678","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.03678/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9QEB1ejx+Y/OFKtkJEE3m0iGbVAuWgOmlyRft8CSMCE27y7eKUpdF5Gri1ofNwiYHMwAZCckzrN0xL1J4YtWBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T01:30:50.962702Z"},"content_sha256":"fa673e5143021f70583c3fa12b779715c5d671f2409f645b290804e89c483f92","schema_version":"1.0","event_id":"sha256:fa673e5143021f70583c3fa12b779715c5d671f2409f645b290804e89c483f92"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/bundle.json","state_url":"https://pith.science/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T01:30:50Z","links":{"resolver":"https://pith.science/pith/IAJI7FIYDI3AQEMITO7AUPW7DG","bundle":"https://pith.science/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/bundle.json","state":"https://pith.science/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IAJI7FIYDI3AQEMITO7AUPW7DG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IAJI7FIYDI3AQEMITO7AUPW7DG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fda304c5ad44b0b628a4771a874940186e399b0fae6a44bbcfcf51dab2aa304b","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T06:52:48Z","title_canon_sha256":"60e020aa2bbd4221e5b4ed0bb2b1363fba3dfa0bd20682637815eb3ef95f4db7"},"schema_version":"1.0","source":{"id":"2407.03678","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.03678","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"arxiv_version","alias_value":"2407.03678v1","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.03678","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_12","alias_value":"IAJI7FIYDI3A","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_16","alias_value":"IAJI7FIYDI3AQEMI","created_at":"2026-07-05T08:40:21Z"},{"alias_kind":"pith_short_8","alias_value":"IAJI7FIY","created_at":"2026-07-05T08:40:21Z"}],"graph_snapshots":[{"event_id":"sha256:fa673e5143021f70583c3fa12b779715c5d671f2409f645b290804e89c483f92","target":"graph","created_at":"2026-07-05T08:40:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.03678/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Prior research has demonstrated noticeable performance gains through the use of probabilistic tokenizations, an approach that involves employing multiple tokenizations of the same input string during the training phase of a language model. Despite these promising findings, modern large language models (LLMs) have yet to be trained using probabilistic tokenizations. Interestingly, while the tokenizers of these contemporary LLMs have the capability to generate multiple tokenizations, this property remains underutilized.\n  In this work, we propose a novel method to leverage the multiple tokenizat","authors_text":"Ashutosh Sathe, Divyanshu Aggarwal, Sunayana Sitaram","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T06:52:48Z","title":"Improving Self Consistency in LLMs through Probabilistic Tokenization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.03678","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:94c7d46d5164aa22d66b77e5d4c788d0363689701e6d46f2b0bfd7635355529a","target":"record","created_at":"2026-07-05T08:40:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fda304c5ad44b0b628a4771a874940186e399b0fae6a44bbcfcf51dab2aa304b","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T06:52:48Z","title_canon_sha256":"60e020aa2bbd4221e5b4ed0bb2b1363fba3dfa0bd20682637815eb3ef95f4db7"},"schema_version":"1.0","source":{"id":"2407.03678","kind":"arxiv","version":1}},"canonical_sha256":"40128f95181a360811889bbe0a3edf19aa4d8fe14c4fc4b3d1a82c76f1af114e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"40128f95181a360811889bbe0a3edf19aa4d8fe14c4fc4b3d1a82c76f1af114e","first_computed_at":"2026-07-05T08:40:21.928292Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:40:21.928292Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X76hvtU6sw0V7PO4PF82RDwMKz3zZ9pPjQLm/35JxTHzCMouevcx02n5ROa/39clToxUQb8WDlbAiehe7EGNDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:40:21.928787Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.03678","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:94c7d46d5164aa22d66b77e5d4c788d0363689701e6d46f2b0bfd7635355529a","sha256:fa673e5143021f70583c3fa12b779715c5d671f2409f645b290804e89c483f92"],"state_sha256":"81c3f9bbee1b393ad05da65a5abeca43ee4211b11484cd45614a18d45c743642"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ltS/MfDH6VSWobj28xivJ7OX4N7odagrjq+eIrrhZ93/1Dh/FdpVbc9R7QXSR14qi7X6Y/ajq5dDaXs6flbxAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T01:30:50.966171Z","bundle_sha256":"4566d269dbc74ac83299bc73a1e6dbe50ba67c67c6e940a458ad3787b0d94863"}}