{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:OFIOFJXDFDYUKRN67TVLJPWLXN","short_pith_number":"pith:OFIOFJXD","canonical_record":{"source":{"id":"2504.16977","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T17:28:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9b750e8fc04fad5d6f0fd69d36f9afa1465949a6e0c4944f5850d02886eca59f","abstract_canon_sha256":"2b9a43277938287e2ed06c7544a53e1c105e2682d009c550ce5714a4d5b1450e"},"schema_version":"1.0"},"canonical_sha256":"7150e2a6e328f14545befceab4becbbb4adaf7edf14d4d1ee3a68cc02051c712","source":{"kind":"arxiv","id":"2504.16977","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.16977","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"arxiv_version","alias_value":"2504.16977v1","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.16977","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_12","alias_value":"OFIOFJXDFDYU","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_16","alias_value":"OFIOFJXDFDYUKRN6","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_8","alias_value":"OFIOFJXD","created_at":"2026-07-05T10:53:20Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:OFIOFJXDFDYUKRN67TVLJPWLXN","target":"record","payload":{"canonical_record":{"source":{"id":"2504.16977","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T17:28:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9b750e8fc04fad5d6f0fd69d36f9afa1465949a6e0c4944f5850d02886eca59f","abstract_canon_sha256":"2b9a43277938287e2ed06c7544a53e1c105e2682d009c550ce5714a4d5b1450e"},"schema_version":"1.0"},"canonical_sha256":"7150e2a6e328f14545befceab4becbbb4adaf7edf14d4d1ee3a68cc02051c712","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:53:20.070874Z","signature_b64":"ANmpXAeyiXg3Bmn/bIJkRshoqlDpxVqojAmg4nOZlhcb4WfRF2KfoohSvVfPmdDmJxPmpSpeZ3TkOzx96CPPCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7150e2a6e328f14545befceab4becbbb4adaf7edf14d4d1ee3a68cc02051c712","last_reissued_at":"2026-07-05T10:53:20.070411Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:53:20.070411Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.16977","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:53:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zzqWBPTxvASEBp8EWKkPKu3K/tWKCvhqcJgH6jcRi5T9erKhZAu2vH0zyzv9DBJRQEgWzEDlp2VoapkBXMNZBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:57:37.774781Z"},"content_sha256":"7936dd9173be552c0892acc1698a6e47dc4ce96b0cb59fa39aef0508fbd1b6fe","schema_version":"1.0","event_id":"sha256:7936dd9173be552c0892acc1698a6e47dc4ce96b0cb59fa39aef0508fbd1b6fe"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:OFIOFJXDFDYUKRN67TVLJPWLXN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Tokenization Matters: Improving Zero-Shot NER for Indic Languages","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Amit Agarwal, Hitesh Laxmichand Patel, Priyaranjan Pattnayak","submitted_at":"2025-04-23T17:28:38Z","abstract_excerpt":"Tokenization is a critical component of Natural Language Processing (NLP), especially for low resource languages, where subword segmentation influences vocabulary structure and downstream task accuracy. Although Byte Pair Encoding (BPE) is a standard tokenization method in multilingual language models, its suitability for Named Entity Recognition (NER) in low resource Indic languages remains underexplored due to its limitations in handling morphological complexity. In this work, we systematically compare BPE, SentencePiece, and Character Level tokenization strategies using IndicBERT for NER ta"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.16977","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.16977/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:53:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uxoffHjbiF1bjNB5xcrNj3+bsaPqmluJWDKjyG96u5ck1L83ClgNCAr5SkPBtyT1FBMxbfZRMyLomQfIjLDLAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:57:37.776061Z"},"content_sha256":"44c7f333905d0f982516912b44a4e379e655a08dfa72302d2134df23742d8b39","schema_version":"1.0","event_id":"sha256:44c7f333905d0f982516912b44a4e379e655a08dfa72302d2134df23742d8b39"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/bundle.json","state_url":"https://pith.science/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T17:57:37Z","links":{"resolver":"https://pith.science/pith/OFIOFJXDFDYUKRN67TVLJPWLXN","bundle":"https://pith.science/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/bundle.json","state":"https://pith.science/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OFIOFJXDFDYUKRN67TVLJPWLXN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:OFIOFJXDFDYUKRN67TVLJPWLXN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2b9a43277938287e2ed06c7544a53e1c105e2682d009c550ce5714a4d5b1450e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T17:28:38Z","title_canon_sha256":"9b750e8fc04fad5d6f0fd69d36f9afa1465949a6e0c4944f5850d02886eca59f"},"schema_version":"1.0","source":{"id":"2504.16977","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.16977","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"arxiv_version","alias_value":"2504.16977v1","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.16977","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_12","alias_value":"OFIOFJXDFDYU","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_16","alias_value":"OFIOFJXDFDYUKRN6","created_at":"2026-07-05T10:53:20Z"},{"alias_kind":"pith_short_8","alias_value":"OFIOFJXD","created_at":"2026-07-05T10:53:20Z"}],"graph_snapshots":[{"event_id":"sha256:44c7f333905d0f982516912b44a4e379e655a08dfa72302d2134df23742d8b39","target":"graph","created_at":"2026-07-05T10:53:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.16977/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tokenization is a critical component of Natural Language Processing (NLP), especially for low resource languages, where subword segmentation influences vocabulary structure and downstream task accuracy. Although Byte Pair Encoding (BPE) is a standard tokenization method in multilingual language models, its suitability for Named Entity Recognition (NER) in low resource Indic languages remains underexplored due to its limitations in handling morphological complexity. In this work, we systematically compare BPE, SentencePiece, and Character Level tokenization strategies using IndicBERT for NER ta","authors_text":"Amit Agarwal, Hitesh Laxmichand Patel, Priyaranjan Pattnayak","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T17:28:38Z","title":"Tokenization Matters: Improving Zero-Shot NER for Indic Languages"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.16977","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7936dd9173be552c0892acc1698a6e47dc4ce96b0cb59fa39aef0508fbd1b6fe","target":"record","created_at":"2026-07-05T10:53:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2b9a43277938287e2ed06c7544a53e1c105e2682d009c550ce5714a4d5b1450e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T17:28:38Z","title_canon_sha256":"9b750e8fc04fad5d6f0fd69d36f9afa1465949a6e0c4944f5850d02886eca59f"},"schema_version":"1.0","source":{"id":"2504.16977","kind":"arxiv","version":1}},"canonical_sha256":"7150e2a6e328f14545befceab4becbbb4adaf7edf14d4d1ee3a68cc02051c712","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7150e2a6e328f14545befceab4becbbb4adaf7edf14d4d1ee3a68cc02051c712","first_computed_at":"2026-07-05T10:53:20.070411Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:53:20.070411Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ANmpXAeyiXg3Bmn/bIJkRshoqlDpxVqojAmg4nOZlhcb4WfRF2KfoohSvVfPmdDmJxPmpSpeZ3TkOzx96CPPCw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:53:20.070874Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.16977","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7936dd9173be552c0892acc1698a6e47dc4ce96b0cb59fa39aef0508fbd1b6fe","sha256:44c7f333905d0f982516912b44a4e379e655a08dfa72302d2134df23742d8b39"],"state_sha256":"008364ac43569aad5289514a8503bfcec89b49e6c49854d68e849a23feb677fb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3+C+4ZiFdrwDqTfYO6chDZ/u1+wvMR3/9+aFrMzT1nfEYKqFSX2wF3vPCPs259/wDd3phM7DcwoFl5RRqC51DQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T17:57:37.783214Z","bundle_sha256":"3b169cd9b22852477b2c5f5bfcd6c73f45849d2e4e50998badaa7dbfed14f225"}}