{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:LBZSB3XBBK2X4MVMQG27RUXLWZ","short_pith_number":"pith:LBZSB3XB","canonical_record":{"source":{"id":"2410.05021","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-07T13:24:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2599f92fb0db4e054e38f9fdc6c2990eed78618cf4cfa3d95cbc77110d6fb7e4","abstract_canon_sha256":"c4c83415174663a6de9eaa54290680b13159b2ee2ec0ee9b28cb76f39f57e9e7"},"schema_version":"1.0"},"canonical_sha256":"587320eee10ab57e32ac81b5f8d2ebb64c4c4176603beaf67613f5bc4f85706d","source":{"kind":"arxiv","id":"2410.05021","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05021","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05021v5","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05021","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_12","alias_value":"LBZSB3XBBK2X","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_16","alias_value":"LBZSB3XBBK2X4MVM","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_8","alias_value":"LBZSB3XB","created_at":"2026-07-05T10:45:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:LBZSB3XBBK2X4MVMQG27RUXLWZ","target":"record","payload":{"canonical_record":{"source":{"id":"2410.05021","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-07T13:24:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2599f92fb0db4e054e38f9fdc6c2990eed78618cf4cfa3d95cbc77110d6fb7e4","abstract_canon_sha256":"c4c83415174663a6de9eaa54290680b13159b2ee2ec0ee9b28cb76f39f57e9e7"},"schema_version":"1.0"},"canonical_sha256":"587320eee10ab57e32ac81b5f8d2ebb64c4c4176603beaf67613f5bc4f85706d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:45:12.278057Z","signature_b64":"HmCzdDqvMnNzbI3feLT9kQorHSfULPd3z+8zKsMX1A02mUG59ocaNmxOtT+L/jS2InXobQRSgzliuIuub2vZDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"587320eee10ab57e32ac81b5f8d2ebb64c4c4176603beaf67613f5bc4f85706d","last_reissued_at":"2026-07-05T10:45:12.277572Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:45:12.277572Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.05021","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:45:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4c/YaUG1a8XvwxPaAOe6pfLzDvJ9LV5svr54K2h9KsaiU4kJGGNXynGo4X49moU7x+FIcEKTgSklpyNtpqKACw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T21:00:30.092870Z"},"content_sha256":"46f3d0bef8c318c38029686dbff780b0733482786302017379a680992cc10621","schema_version":"1.0","event_id":"sha256:46f3d0bef8c318c38029686dbff780b0733482786302017379a680992cc10621"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:LBZSB3XBBK2X4MVMQG27RUXLWZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"DEPT: Decoupled Embeddings for Pre-training Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Alex Iacob, Dongqi Cai, Lorenzo Sani, Meghdad Kurmanji, Nicholas D. Lane, William F. Shen, Xinchi Qiu, Yan Gao","submitted_at":"2024-10-07T13:24:24Z","abstract_excerpt":"Language Model pre-training uses broad data mixtures to enhance performance across domains and languages. However, training on such heterogeneous text corpora requires extensive and expensive efforts. Since these data sources vary significantly in lexical, syntactic, and semantic aspects, they cause negative interference or the ``curse of multilinguality''. To address these challenges we propose a communication-efficient pre-training framework, DEPT. Our method decouples embeddings from the transformer body while simultaneously training the latter on multiple data sources without requiring a s"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05021","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.05021/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:45:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YCppw5p4JLajxhMPhq+pBq8bD5OP4AYrH1eMWw1azn/1asRy4lfxPJZzIepTNnvJzZIu/+Wm0mz3FtV9lByYCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T21:00:30.093570Z"},"content_sha256":"c161f9d6a5e0c1e4894647f3eb77ca129fe691ea2011cc117c338084e5f802f2","schema_version":"1.0","event_id":"sha256:c161f9d6a5e0c1e4894647f3eb77ca129fe691ea2011cc117c338084e5f802f2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/bundle.json","state_url":"https://pith.science/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T21:00:30Z","links":{"resolver":"https://pith.science/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ","bundle":"https://pith.science/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/bundle.json","state":"https://pith.science/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LBZSB3XBBK2X4MVMQG27RUXLWZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LBZSB3XBBK2X4MVMQG27RUXLWZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c4c83415174663a6de9eaa54290680b13159b2ee2ec0ee9b28cb76f39f57e9e7","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-07T13:24:24Z","title_canon_sha256":"2599f92fb0db4e054e38f9fdc6c2990eed78618cf4cfa3d95cbc77110d6fb7e4"},"schema_version":"1.0","source":{"id":"2410.05021","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05021","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05021v5","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05021","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_12","alias_value":"LBZSB3XBBK2X","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_16","alias_value":"LBZSB3XBBK2X4MVM","created_at":"2026-07-05T10:45:12Z"},{"alias_kind":"pith_short_8","alias_value":"LBZSB3XB","created_at":"2026-07-05T10:45:12Z"}],"graph_snapshots":[{"event_id":"sha256:c161f9d6a5e0c1e4894647f3eb77ca129fe691ea2011cc117c338084e5f802f2","target":"graph","created_at":"2026-07-05T10:45:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.05021/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language Model pre-training uses broad data mixtures to enhance performance across domains and languages. However, training on such heterogeneous text corpora requires extensive and expensive efforts. Since these data sources vary significantly in lexical, syntactic, and semantic aspects, they cause negative interference or the ``curse of multilinguality''. To address these challenges we propose a communication-efficient pre-training framework, DEPT. Our method decouples embeddings from the transformer body while simultaneously training the latter on multiple data sources without requiring a s","authors_text":"Alex Iacob, Dongqi Cai, Lorenzo Sani, Meghdad Kurmanji, Nicholas D. Lane, William F. Shen, Xinchi Qiu, Yan Gao","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-07T13:24:24Z","title":"DEPT: Decoupled Embeddings for Pre-training Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05021","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:46f3d0bef8c318c38029686dbff780b0733482786302017379a680992cc10621","target":"record","created_at":"2026-07-05T10:45:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c4c83415174663a6de9eaa54290680b13159b2ee2ec0ee9b28cb76f39f57e9e7","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-07T13:24:24Z","title_canon_sha256":"2599f92fb0db4e054e38f9fdc6c2990eed78618cf4cfa3d95cbc77110d6fb7e4"},"schema_version":"1.0","source":{"id":"2410.05021","kind":"arxiv","version":5}},"canonical_sha256":"587320eee10ab57e32ac81b5f8d2ebb64c4c4176603beaf67613f5bc4f85706d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"587320eee10ab57e32ac81b5f8d2ebb64c4c4176603beaf67613f5bc4f85706d","first_computed_at":"2026-07-05T10:45:12.277572Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:45:12.277572Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"HmCzdDqvMnNzbI3feLT9kQorHSfULPd3z+8zKsMX1A02mUG59ocaNmxOtT+L/jS2InXobQRSgzliuIuub2vZDg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:45:12.278057Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.05021","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:46f3d0bef8c318c38029686dbff780b0733482786302017379a680992cc10621","sha256:c161f9d6a5e0c1e4894647f3eb77ca129fe691ea2011cc117c338084e5f802f2"],"state_sha256":"4bda317a66d5e9688f3ebefbdf454bd054d22c2719f97b7a96953ac3dbb4b908"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"h+h8mLdfpz/jghtzMLAqRCiPeXgf8+3iPBWISQAGCFbVpVtfqqAnY18/63VGWeTn5x0sNrpAyuIrlW6DH0qXBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T21:00:30.100408Z","bundle_sha256":"89b5f79d5725cc52d85ffd9e03c390b0068560c71e676138d4c2d461ad0779d6"}}