{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:RT6PNZRJ42OC5AJGIGWEYMZTCV","short_pith_number":"pith:RT6PNZRJ","canonical_record":{"source":{"id":"2010.01825","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-05T07:19:52Z","cross_cats_sorted":["cs.CL","stat.ML"],"title_canon_sha256":"f065d4b5fca057fe975c8ba4133a5a97d669a2e4f99c87aaaf3d06bf183c7b23","abstract_canon_sha256":"414a97e00d602f482a482c3d750196205f931a330f3ae81c088ccc27da527463"},"schema_version":"1.0"},"canonical_sha256":"8cfcf6e629e69c2e812641ac4c3333156e48d708a25238c3c99c05910fe31b02","source":{"kind":"arxiv","id":"2010.01825","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.01825","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"arxiv_version","alias_value":"2010.01825v1","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.01825","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_12","alias_value":"RT6PNZRJ42OC","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_16","alias_value":"RT6PNZRJ42OC5AJG","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_8","alias_value":"RT6PNZRJ","created_at":"2026-07-05T01:40:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:RT6PNZRJ42OC5AJGIGWEYMZTCV","target":"record","payload":{"canonical_record":{"source":{"id":"2010.01825","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-05T07:19:52Z","cross_cats_sorted":["cs.CL","stat.ML"],"title_canon_sha256":"f065d4b5fca057fe975c8ba4133a5a97d669a2e4f99c87aaaf3d06bf183c7b23","abstract_canon_sha256":"414a97e00d602f482a482c3d750196205f931a330f3ae81c088ccc27da527463"},"schema_version":"1.0"},"canonical_sha256":"8cfcf6e629e69c2e812641ac4c3333156e48d708a25238c3c99c05910fe31b02","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:40:09.940254Z","signature_b64":"gA7FzrlUkquEr8yc9FZMA0fCd0AlupF91A5rBPkODW4RYWHJW/n0KaTAU/WYb6yE3kZNhc84jLyNmJlQimOSAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8cfcf6e629e69c2e812641ac4c3333156e48d708a25238c3c99c05910fe31b02","last_reissued_at":"2026-07-05T01:40:09.939863Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:40:09.939863Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2010.01825","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uXGVCmn00FhUL3VD1qmQyp4YrVdBDJ8Y7b+IXX4Aj+k4iivsSSIxqDLervT2sHMbs884vFoP7Z35qFEQIoV2CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T15:17:40.997595Z"},"content_sha256":"f809096119abea3df9bedd60c9e405c8b6759553fb995d4ef3d819f9892c1236","schema_version":"1.0","event_id":"sha256:f809096119abea3df9bedd60c9e405c8b6759553fb995d4ef3d819f9892c1236"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:RT6PNZRJ42OC5AJGIGWEYMZTCV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PMI-Masking: Principled masking of correlated spans","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Barak Lenz, Kevin Leyton-Brown, Moshe Tennenholtz, Omri Abend, Opher Lieber, Yoav Levine, Yoav Shoham","submitted_at":"2020-10-05T07:19:52Z","abstract_excerpt":"Masking tokens uniformly at random constitutes a common flaw in the pretraining of Masked Language Models (MLMs) such as BERT. We show that such uniform masking allows an MLM to minimize its training objective by latching onto shallow local signals, leading to pretraining inefficiency and suboptimal downstream performance. To address this flaw, we propose PMI-Masking, a principled masking strategy based on the concept of Pointwise Mutual Information (PMI), which jointly masks a token n-gram if it exhibits high collocation over the corpus. PMI-Masking motivates, unifies, and improves upon prior"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.01825","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.01825/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fknhPTH0PWv8oldvA4nSIJhnygUzCZPwN/Yo06A/lV68d7PlvkfvIn4rTB+f/IVnZOYd9OvpYSZ+4lH8QLDlAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T15:17:40.998091Z"},"content_sha256":"a1740dee098ac0a9c5d6c2b7ba0af2f0b6671ca52a3bed9b2fb899ee357c301c","schema_version":"1.0","event_id":"sha256:a1740dee098ac0a9c5d6c2b7ba0af2f0b6671ca52a3bed9b2fb899ee357c301c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/bundle.json","state_url":"https://pith.science/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T15:17:41Z","links":{"resolver":"https://pith.science/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV","bundle":"https://pith.science/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/bundle.json","state":"https://pith.science/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RT6PNZRJ42OC5AJGIGWEYMZTCV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:RT6PNZRJ42OC5AJGIGWEYMZTCV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"414a97e00d602f482a482c3d750196205f931a330f3ae81c088ccc27da527463","cross_cats_sorted":["cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-05T07:19:52Z","title_canon_sha256":"f065d4b5fca057fe975c8ba4133a5a97d669a2e4f99c87aaaf3d06bf183c7b23"},"schema_version":"1.0","source":{"id":"2010.01825","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.01825","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"arxiv_version","alias_value":"2010.01825v1","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.01825","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_12","alias_value":"RT6PNZRJ42OC","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_16","alias_value":"RT6PNZRJ42OC5AJG","created_at":"2026-07-05T01:40:09Z"},{"alias_kind":"pith_short_8","alias_value":"RT6PNZRJ","created_at":"2026-07-05T01:40:09Z"}],"graph_snapshots":[{"event_id":"sha256:a1740dee098ac0a9c5d6c2b7ba0af2f0b6671ca52a3bed9b2fb899ee357c301c","target":"graph","created_at":"2026-07-05T01:40:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2010.01825/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Masking tokens uniformly at random constitutes a common flaw in the pretraining of Masked Language Models (MLMs) such as BERT. We show that such uniform masking allows an MLM to minimize its training objective by latching onto shallow local signals, leading to pretraining inefficiency and suboptimal downstream performance. To address this flaw, we propose PMI-Masking, a principled masking strategy based on the concept of Pointwise Mutual Information (PMI), which jointly masks a token n-gram if it exhibits high collocation over the corpus. PMI-Masking motivates, unifies, and improves upon prior","authors_text":"Barak Lenz, Kevin Leyton-Brown, Moshe Tennenholtz, Omri Abend, Opher Lieber, Yoav Levine, Yoav Shoham","cross_cats":["cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-05T07:19:52Z","title":"PMI-Masking: Principled masking of correlated spans"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.01825","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f809096119abea3df9bedd60c9e405c8b6759553fb995d4ef3d819f9892c1236","target":"record","created_at":"2026-07-05T01:40:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"414a97e00d602f482a482c3d750196205f931a330f3ae81c088ccc27da527463","cross_cats_sorted":["cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-05T07:19:52Z","title_canon_sha256":"f065d4b5fca057fe975c8ba4133a5a97d669a2e4f99c87aaaf3d06bf183c7b23"},"schema_version":"1.0","source":{"id":"2010.01825","kind":"arxiv","version":1}},"canonical_sha256":"8cfcf6e629e69c2e812641ac4c3333156e48d708a25238c3c99c05910fe31b02","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8cfcf6e629e69c2e812641ac4c3333156e48d708a25238c3c99c05910fe31b02","first_computed_at":"2026-07-05T01:40:09.939863Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:40:09.939863Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gA7FzrlUkquEr8yc9FZMA0fCd0AlupF91A5rBPkODW4RYWHJW/n0KaTAU/WYb6yE3kZNhc84jLyNmJlQimOSAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T01:40:09.940254Z","signed_message":"canonical_sha256_bytes"},"source_id":"2010.01825","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f809096119abea3df9bedd60c9e405c8b6759553fb995d4ef3d819f9892c1236","sha256:a1740dee098ac0a9c5d6c2b7ba0af2f0b6671ca52a3bed9b2fb899ee357c301c"],"state_sha256":"941d37249d83b0de50c70f7f3fbefdf343baa46dbf596bdd3780c86578828a66"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PxWIA90TAAQObodY3JuEpEPtZq6I/UFCzB7qx8BA5mouHDh+D/0htsNd94wVjVnSbwTb5le5xrfKU2u+pa5sDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T15:17:41.001923Z","bundle_sha256":"271ce4544aa095942beb6e5e9411bff7e6bc09aa9eff13181cef569f5f2f1073"}}