{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:K2NL2NXMCGQZ3BKNNSR5ACRGV3","short_pith_number":"pith:K2NL2NXM","canonical_record":{"source":{"id":"2305.15273","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T15:59:44Z","cross_cats_sorted":[],"title_canon_sha256":"437f764bc9e45e9be41e9d8825f3b920447a5794ebaecde8d8aa0a45f1c37d5f","abstract_canon_sha256":"552df4b3b7b42dc97e907740feb9ae0ce39941372ea171c83106f07efa6cdbdc"},"schema_version":"1.0"},"canonical_sha256":"569abd36ec11a19d854d6ca3d00a26aec4b4f917e852a46538914a58041087c4","source":{"kind":"arxiv","id":"2305.15273","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.15273","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"arxiv_version","alias_value":"2305.15273v1","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.15273","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_12","alias_value":"K2NL2NXMCGQZ","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_16","alias_value":"K2NL2NXMCGQZ3BKN","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_8","alias_value":"K2NL2NXM","created_at":"2026-07-05T06:13:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:K2NL2NXMCGQZ3BKNNSR5ACRGV3","target":"record","payload":{"canonical_record":{"source":{"id":"2305.15273","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T15:59:44Z","cross_cats_sorted":[],"title_canon_sha256":"437f764bc9e45e9be41e9d8825f3b920447a5794ebaecde8d8aa0a45f1c37d5f","abstract_canon_sha256":"552df4b3b7b42dc97e907740feb9ae0ce39941372ea171c83106f07efa6cdbdc"},"schema_version":"1.0"},"canonical_sha256":"569abd36ec11a19d854d6ca3d00a26aec4b4f917e852a46538914a58041087c4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:13:33.218162Z","signature_b64":"f+jNf0AscICKnSIBDfNKb3XMOENDabYJZN/use018K5ffSo85wYjKeSlTo0jQ1ZpYogVHhEeZNGGBOLLWvEFBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"569abd36ec11a19d854d6ca3d00a26aec4b4f917e852a46538914a58041087c4","last_reissued_at":"2026-07-05T06:13:33.217755Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:13:33.217755Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.15273","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:13:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sNQMZRP32nUUCJc86GXshSqLVefps6XI4dnlr055XT4oKPnVrAzRkxZZTwvRdsjNTS4D+kXSeP36dgj+v3oCBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T13:44:54.557964Z"},"content_sha256":"5727260b94a2133520ff53b8cb2bea72758c3c2c04975b9c5ef409dc8d02bba2","schema_version":"1.0","event_id":"sha256:5727260b94a2133520ff53b8cb2bea72758c3c2c04975b9c5ef409dc8d02bba2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:K2NL2NXMCGQZ3BKNNSR5ACRGV3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Revisiting Token Dropping Strategy in Efficient BERT Pretraining","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Bo Du, Dacheng Tao, Juhua Liu, Liang Ding, Min Zhang, Qihuang Zhong, Xuebo Liu","submitted_at":"2023-05-24T15:59:44Z","abstract_excerpt":"Token dropping is a recently-proposed strategy to speed up the pretraining of masked language models, such as BERT, by skipping the computation of a subset of the input tokens at several middle layers. It can effectively reduce the training time without degrading much performance on downstream tasks. However, we empirically find that token dropping is prone to a semantic loss problem and falls short in handling semantic-intense tasks. Motivated by this, we propose a simple yet effective semantic-consistent learning method (ScTD) to improve the token dropping. ScTD aims to encourage the model t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.15273","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.15273/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:13:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oNWpMpQRQuyfpGLTW6WqgVVUyNwFWUnCeG2fTjJBkHvBoMlafWdBmqofCTrcb8Q/WQowpooxSzCxH3coT3UVCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T13:44:54.558700Z"},"content_sha256":"765c1f5e061f70cefd982cd3e99a6716db0e53738bbda1e43bdfb6f8e3d29761","schema_version":"1.0","event_id":"sha256:765c1f5e061f70cefd982cd3e99a6716db0e53738bbda1e43bdfb6f8e3d29761"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/bundle.json","state_url":"https://pith.science/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T13:44:54Z","links":{"resolver":"https://pith.science/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3","bundle":"https://pith.science/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/bundle.json","state":"https://pith.science/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/K2NL2NXMCGQZ3BKNNSR5ACRGV3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:K2NL2NXMCGQZ3BKNNSR5ACRGV3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"552df4b3b7b42dc97e907740feb9ae0ce39941372ea171c83106f07efa6cdbdc","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T15:59:44Z","title_canon_sha256":"437f764bc9e45e9be41e9d8825f3b920447a5794ebaecde8d8aa0a45f1c37d5f"},"schema_version":"1.0","source":{"id":"2305.15273","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.15273","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"arxiv_version","alias_value":"2305.15273v1","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.15273","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_12","alias_value":"K2NL2NXMCGQZ","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_16","alias_value":"K2NL2NXMCGQZ3BKN","created_at":"2026-07-05T06:13:33Z"},{"alias_kind":"pith_short_8","alias_value":"K2NL2NXM","created_at":"2026-07-05T06:13:33Z"}],"graph_snapshots":[{"event_id":"sha256:765c1f5e061f70cefd982cd3e99a6716db0e53738bbda1e43bdfb6f8e3d29761","target":"graph","created_at":"2026-07-05T06:13:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.15273/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Token dropping is a recently-proposed strategy to speed up the pretraining of masked language models, such as BERT, by skipping the computation of a subset of the input tokens at several middle layers. It can effectively reduce the training time without degrading much performance on downstream tasks. However, we empirically find that token dropping is prone to a semantic loss problem and falls short in handling semantic-intense tasks. Motivated by this, we propose a simple yet effective semantic-consistent learning method (ScTD) to improve the token dropping. ScTD aims to encourage the model t","authors_text":"Bo Du, Dacheng Tao, Juhua Liu, Liang Ding, Min Zhang, Qihuang Zhong, Xuebo Liu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T15:59:44Z","title":"Revisiting Token Dropping Strategy in Efficient BERT Pretraining"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.15273","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5727260b94a2133520ff53b8cb2bea72758c3c2c04975b9c5ef409dc8d02bba2","target":"record","created_at":"2026-07-05T06:13:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"552df4b3b7b42dc97e907740feb9ae0ce39941372ea171c83106f07efa6cdbdc","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T15:59:44Z","title_canon_sha256":"437f764bc9e45e9be41e9d8825f3b920447a5794ebaecde8d8aa0a45f1c37d5f"},"schema_version":"1.0","source":{"id":"2305.15273","kind":"arxiv","version":1}},"canonical_sha256":"569abd36ec11a19d854d6ca3d00a26aec4b4f917e852a46538914a58041087c4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"569abd36ec11a19d854d6ca3d00a26aec4b4f917e852a46538914a58041087c4","first_computed_at":"2026-07-05T06:13:33.217755Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:13:33.217755Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"f+jNf0AscICKnSIBDfNKb3XMOENDabYJZN/use018K5ffSo85wYjKeSlTo0jQ1ZpYogVHhEeZNGGBOLLWvEFBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:13:33.218162Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.15273","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5727260b94a2133520ff53b8cb2bea72758c3c2c04975b9c5ef409dc8d02bba2","sha256:765c1f5e061f70cefd982cd3e99a6716db0e53738bbda1e43bdfb6f8e3d29761"],"state_sha256":"9353abe41ad77ed784ea37836621f6b182edccbbc9df4e20f9487250ff26ed6a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VOTmr7Gm6R0lC8W1p0mCJOcCNHc86GHBvAi/QnMmxaCvWvkcCp3Om6sQBmm8h1jtvVxuHSVDVOITeXf+2x1zAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T13:44:54.565146Z","bundle_sha256":"295d8afd8fd9e8839b8f6a3778b7df7d62c8b1ecc55660d89bc8fe7b51b161ff"}}