{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:L3MUMAZ4QYNUXJT64R7ZYOQ4BI","short_pith_number":"pith:L3MUMAZ4","canonical_record":{"source":{"id":"2105.12410","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-26T09:05:10Z","cross_cats_sorted":[],"title_canon_sha256":"bba467d9cada6b3433b870e5867b489c02fd25f5cab3ea34c6827020bcb9c8bf","abstract_canon_sha256":"21c59e666d38bbd4c47d61c71d315bf8f44bb121b83df0c2a99e2ae82dca94c1"},"schema_version":"1.0"},"canonical_sha256":"5ed946033c861b4ba67ee47f9c3a1c0a162b4189f2432d148fe9f33c09f738cf","source":{"kind":"arxiv","id":"2105.12410","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.12410","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"arxiv_version","alias_value":"2105.12410v1","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.12410","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_12","alias_value":"L3MUMAZ4QYNU","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_16","alias_value":"L3MUMAZ4QYNUXJT6","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_8","alias_value":"L3MUMAZ4","created_at":"2026-07-05T02:43:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:L3MUMAZ4QYNUXJT64R7ZYOQ4BI","target":"record","payload":{"canonical_record":{"source":{"id":"2105.12410","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-26T09:05:10Z","cross_cats_sorted":[],"title_canon_sha256":"bba467d9cada6b3433b870e5867b489c02fd25f5cab3ea34c6827020bcb9c8bf","abstract_canon_sha256":"21c59e666d38bbd4c47d61c71d315bf8f44bb121b83df0c2a99e2ae82dca94c1"},"schema_version":"1.0"},"canonical_sha256":"5ed946033c861b4ba67ee47f9c3a1c0a162b4189f2432d148fe9f33c09f738cf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:43:38.560013Z","signature_b64":"u8cjVGDAkmyXGAFvq4z99KyFkqcE3LKlu/lygBS4pKnWZnidN8HGt70NyvDOIwvm/y92UkiSAlCf0bEzqz83DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5ed946033c861b4ba67ee47f9c3a1c0a162b4189f2432d148fe9f33c09f738cf","last_reissued_at":"2026-07-05T02:43:38.559591Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:43:38.559591Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2105.12410","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:43:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kIwawgHsPgGgXUIOLdTai6vGZUd8geTQyXOOpB5SkPQRd2blA9/sQzAsCcME59tcVjkqcrFV8iOAL2wATdXcBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T08:10:29.244190Z"},"content_sha256":"fcc2225ddecf5d8e4825146f2566e725de2d8f33c381473f9cb0d5ad078a598a","schema_version":"1.0","event_id":"sha256:fcc2225ddecf5d8e4825146f2566e725de2d8f33c381473f9cb0d5ad078a598a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:L3MUMAZ4QYNUXJT64R7ZYOQ4BI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Joint Optimization of Tokenization and Downstream Model","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Atsushi Keyaki, Kei Uchiumi, Naoaki Okazaki, Sho Takase, Tatsuya Hiraoka","submitted_at":"2021-05-26T09:05:10Z","abstract_excerpt":"Since traditional tokenizers are isolated from a downstream task and model, they cannot output an appropriate tokenization depending on the task and model, although recent studies imply that the appropriate tokenization improves the performance. In this paper, we propose a novel method to find an appropriate tokenization to a given downstream model by jointly optimizing a tokenizer and the model. The proposed method has no restriction except for using loss values computed by the downstream model to train the tokenizer, and thus, we can apply the proposed method to any NLP task. Moreover, the p"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.12410","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.12410/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:43:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"svhc0qDlIiUU9PcrwAkf79zEoyLYQVNMWjMqq7l9F4MipidrHIjuY0zVQZyeBquMgz6yTsxGLEBdHdTB+p31Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T08:10:29.244557Z"},"content_sha256":"8149cb33ddea840001e91b29857dca5e08c3c22378d7ddf2976d5daf3b036f04","schema_version":"1.0","event_id":"sha256:8149cb33ddea840001e91b29857dca5e08c3c22378d7ddf2976d5daf3b036f04"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/bundle.json","state_url":"https://pith.science/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T08:10:29Z","links":{"resolver":"https://pith.science/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI","bundle":"https://pith.science/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/bundle.json","state":"https://pith.science/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/L3MUMAZ4QYNUXJT64R7ZYOQ4BI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:L3MUMAZ4QYNUXJT64R7ZYOQ4BI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"21c59e666d38bbd4c47d61c71d315bf8f44bb121b83df0c2a99e2ae82dca94c1","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-26T09:05:10Z","title_canon_sha256":"bba467d9cada6b3433b870e5867b489c02fd25f5cab3ea34c6827020bcb9c8bf"},"schema_version":"1.0","source":{"id":"2105.12410","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.12410","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"arxiv_version","alias_value":"2105.12410v1","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.12410","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_12","alias_value":"L3MUMAZ4QYNU","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_16","alias_value":"L3MUMAZ4QYNUXJT6","created_at":"2026-07-05T02:43:38Z"},{"alias_kind":"pith_short_8","alias_value":"L3MUMAZ4","created_at":"2026-07-05T02:43:38Z"}],"graph_snapshots":[{"event_id":"sha256:8149cb33ddea840001e91b29857dca5e08c3c22378d7ddf2976d5daf3b036f04","target":"graph","created_at":"2026-07-05T02:43:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2105.12410/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Since traditional tokenizers are isolated from a downstream task and model, they cannot output an appropriate tokenization depending on the task and model, although recent studies imply that the appropriate tokenization improves the performance. In this paper, we propose a novel method to find an appropriate tokenization to a given downstream model by jointly optimizing a tokenizer and the model. The proposed method has no restriction except for using loss values computed by the downstream model to train the tokenizer, and thus, we can apply the proposed method to any NLP task. Moreover, the p","authors_text":"Atsushi Keyaki, Kei Uchiumi, Naoaki Okazaki, Sho Takase, Tatsuya Hiraoka","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-26T09:05:10Z","title":"Joint Optimization of Tokenization and Downstream Model"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.12410","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fcc2225ddecf5d8e4825146f2566e725de2d8f33c381473f9cb0d5ad078a598a","target":"record","created_at":"2026-07-05T02:43:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"21c59e666d38bbd4c47d61c71d315bf8f44bb121b83df0c2a99e2ae82dca94c1","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-26T09:05:10Z","title_canon_sha256":"bba467d9cada6b3433b870e5867b489c02fd25f5cab3ea34c6827020bcb9c8bf"},"schema_version":"1.0","source":{"id":"2105.12410","kind":"arxiv","version":1}},"canonical_sha256":"5ed946033c861b4ba67ee47f9c3a1c0a162b4189f2432d148fe9f33c09f738cf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5ed946033c861b4ba67ee47f9c3a1c0a162b4189f2432d148fe9f33c09f738cf","first_computed_at":"2026-07-05T02:43:38.559591Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:43:38.559591Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"u8cjVGDAkmyXGAFvq4z99KyFkqcE3LKlu/lygBS4pKnWZnidN8HGt70NyvDOIwvm/y92UkiSAlCf0bEzqz83DA==","signature_status":"signed_v1","signed_at":"2026-07-05T02:43:38.560013Z","signed_message":"canonical_sha256_bytes"},"source_id":"2105.12410","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fcc2225ddecf5d8e4825146f2566e725de2d8f33c381473f9cb0d5ad078a598a","sha256:8149cb33ddea840001e91b29857dca5e08c3c22378d7ddf2976d5daf3b036f04"],"state_sha256":"da6f35a1fbd42c31ab03c28a85ad89e6eceb2e91ae2659b83c76948dc63efa84"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TQxQbEAPFu0meVqMHA/zBnEpk8d89sO+olNiZklRC6aje4a2wKBaLSBxMRU2MAIQRw4x72LyDAmbcCBCvdNzAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T08:10:29.249220Z","bundle_sha256":"00a77627d61adcd2d839d3486fe063ed2eb25e0b4120118009965f32020c9830"}}