{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:RDRB2NHSA3USS5GEDIV7YK3SBZ","short_pith_number":"pith:RDRB2NHS","schema_version":"1.0","canonical_sha256":"88e21d34f206e92974c41a2bfc2b720e6f28168df5959a6a41d210e0fb82c27a","source":{"kind":"arxiv","id":"1908.09716","version":1},"attestation_state":"computed","paper":{"title":"uniblock: Scoring and Filtering Corpus with Unicode Block Information","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hermann Ney, Weiyue Wang, Yingbo Gao","submitted_at":"2019-08-26T14:55:03Z","abstract_excerpt":"The preprocessing pipelines in Natural Language Processing usually involve a step of removing sentences consisted of illegal characters. The definition of illegal characters and the specific removal strategy depend on the task, language, domain, etc, which often lead to tiresome and repetitive scripting of rules. In this paper, we introduce a simple statistical method, uniblock, to overcome this problem. For each sentence, uniblock generates a fixed-size feature vector using Unicode block information of the characters. A Gaussian mixture model is then estimated on some clean corpus using varia"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1908.09716","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-08-26T14:55:03Z","cross_cats_sorted":[],"title_canon_sha256":"8aa8895419db0bb0ca5fefda886e72947066550d606df331b9d9bde4b188ddc1","abstract_canon_sha256":"9d564f47413aff791f1ae2c17f84cc9547f6984e1e0183f0080cf609f528b89f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-04T23:59:42.910960Z","signature_b64":"z319i52xAATWIwS0xe6ptoM/XpYKvIAy0/WzgWf9YDTTGTDAR4TEgLT/A4kZZ0++Q7Q2Xp/7SSEhWuVsOY5WDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"88e21d34f206e92974c41a2bfc2b720e6f28168df5959a6a41d210e0fb82c27a","last_reissued_at":"2026-07-04T23:59:42.910577Z","signature_status":"signed_v1","first_computed_at":"2026-07-04T23:59:42.910577Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"uniblock: Scoring and Filtering Corpus with Unicode Block Information","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hermann Ney, Weiyue Wang, Yingbo Gao","submitted_at":"2019-08-26T14:55:03Z","abstract_excerpt":"The preprocessing pipelines in Natural Language Processing usually involve a step of removing sentences consisted of illegal characters. The definition of illegal characters and the specific removal strategy depend on the task, language, domain, etc, which often lead to tiresome and repetitive scripting of rules. In this paper, we introduce a simple statistical method, uniblock, to overcome this problem. For each sentence, uniblock generates a fixed-size feature vector using Unicode block information of the characters. A Gaussian mixture model is then estimated on some clean corpus using varia"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1908.09716","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1908.09716/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1908.09716","created_at":"2026-07-04T23:59:42.910630+00:00"},{"alias_kind":"arxiv_version","alias_value":"1908.09716v1","created_at":"2026-07-04T23:59:42.910630+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1908.09716","created_at":"2026-07-04T23:59:42.910630+00:00"},{"alias_kind":"pith_short_12","alias_value":"RDRB2NHSA3US","created_at":"2026-07-04T23:59:42.910630+00:00"},{"alias_kind":"pith_short_16","alias_value":"RDRB2NHSA3USS5GE","created_at":"2026-07-04T23:59:42.910630+00:00"},{"alias_kind":"pith_short_8","alias_value":"RDRB2NHS","created_at":"2026-07-04T23:59:42.910630+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ","json":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ.json","graph_json":"https://pith.science/api/pith-number/RDRB2NHSA3USS5GEDIV7YK3SBZ/graph.json","events_json":"https://pith.science/api/pith-number/RDRB2NHSA3USS5GEDIV7YK3SBZ/events.json","paper":"https://pith.science/paper/RDRB2NHS"},"agent_actions":{"view_html":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ","download_json":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ.json","view_paper":"https://pith.science/paper/RDRB2NHS","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1908.09716&json=true","fetch_graph":"https://pith.science/api/pith-number/RDRB2NHSA3USS5GEDIV7YK3SBZ/graph.json","fetch_events":"https://pith.science/api/pith-number/RDRB2NHSA3USS5GEDIV7YK3SBZ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ/action/storage_attestation","attest_author":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ/action/author_attestation","sign_citation":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ/action/citation_signature","submit_replication":"https://pith.science/pith/RDRB2NHSA3USS5GEDIV7YK3SBZ/action/replication_record"}},"created_at":"2026-07-04T23:59:42.910630+00:00","updated_at":"2026-07-04T23:59:42.910630+00:00"}