{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:HAKITHMCCOTEOA6HSAP4M7JADG","short_pith_number":"pith:HAKITHMC","schema_version":"1.0","canonical_sha256":"3814899d8213a64703c7901fc67d2019812e2dcc1af6963ac7b442ec4c079b31","source":{"kind":"arxiv","id":"2005.06166","version":1},"attestation_state":"computed","paper":{"title":"Parallel Corpus Filtering via Pre-trained Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ajay Nagesh, Boliang Zhang, Kevin Knight","submitted_at":"2020-05-13T06:06:23Z","abstract_excerpt":"Web-crawled data provides a good source of parallel corpora for training machine translation models. It is automatically obtained, but extremely noisy, and recent work shows that neural machine translation systems are more sensitive to noise than traditional statistical machine translation methods. In this paper, we propose a novel approach to filter out noisy sentence pairs from web-crawled corpora via pre-trained language models. We measure sentence parallelism by leveraging the multilingual capability of BERT and use the Generative Pre-training (GPT) language model as a domain filter to bal"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2005.06166","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-05-13T06:06:23Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"7f5c5503df6e71846a0d00336e67527a91ccb3c038d0524b4194abef6d4da634","abstract_canon_sha256":"6988c6b73aab244498bf338d082217dd192cfdf535350945c1bdbf127f30afdb"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:02:32.713271Z","signature_b64":"53ZWXvL1Pxcx6VcN6c7MVGEwQltA9eJlE6pDs28lOlv8JWiiZZYamfG0I73fRcflWh8cHw5l6FZv2/WukwllDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3814899d8213a64703c7901fc67d2019812e2dcc1af6963ac7b442ec4c079b31","last_reissued_at":"2026-07-05T01:02:32.712707Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:02:32.712707Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Parallel Corpus Filtering via Pre-trained Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ajay Nagesh, Boliang Zhang, Kevin Knight","submitted_at":"2020-05-13T06:06:23Z","abstract_excerpt":"Web-crawled data provides a good source of parallel corpora for training machine translation models. It is automatically obtained, but extremely noisy, and recent work shows that neural machine translation systems are more sensitive to noise than traditional statistical machine translation methods. In this paper, we propose a novel approach to filter out noisy sentence pairs from web-crawled corpora via pre-trained language models. We measure sentence parallelism by leveraging the multilingual capability of BERT and use the Generative Pre-training (GPT) language model as a domain filter to bal"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.06166","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2005.06166/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2005.06166","created_at":"2026-07-05T01:02:32.712794+00:00"},{"alias_kind":"arxiv_version","alias_value":"2005.06166v1","created_at":"2026-07-05T01:02:32.712794+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.06166","created_at":"2026-07-05T01:02:32.712794+00:00"},{"alias_kind":"pith_short_12","alias_value":"HAKITHMCCOTE","created_at":"2026-07-05T01:02:32.712794+00:00"},{"alias_kind":"pith_short_16","alias_value":"HAKITHMCCOTEOA6H","created_at":"2026-07-05T01:02:32.712794+00:00"},{"alias_kind":"pith_short_8","alias_value":"HAKITHMC","created_at":"2026-07-05T01:02:32.712794+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG","json":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG.json","graph_json":"https://pith.science/api/pith-number/HAKITHMCCOTEOA6HSAP4M7JADG/graph.json","events_json":"https://pith.science/api/pith-number/HAKITHMCCOTEOA6HSAP4M7JADG/events.json","paper":"https://pith.science/paper/HAKITHMC"},"agent_actions":{"view_html":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG","download_json":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG.json","view_paper":"https://pith.science/paper/HAKITHMC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2005.06166&json=true","fetch_graph":"https://pith.science/api/pith-number/HAKITHMCCOTEOA6HSAP4M7JADG/graph.json","fetch_events":"https://pith.science/api/pith-number/HAKITHMCCOTEOA6HSAP4M7JADG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG/action/storage_attestation","attest_author":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG/action/author_attestation","sign_citation":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG/action/citation_signature","submit_replication":"https://pith.science/pith/HAKITHMCCOTEOA6HSAP4M7JADG/action/replication_record"}},"created_at":"2026-07-05T01:02:32.712794+00:00","updated_at":"2026-07-05T01:02:32.712794+00:00"}