{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:YTAAWKWITQI43IH67TZXPIVRAR","short_pith_number":"pith:YTAAWKWI","canonical_record":{"source":{"id":"2309.08351","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-15T12:20:00Z","cross_cats_sorted":[],"title_canon_sha256":"3ab4e5ddc4d124b37a9f07d23b799844cccdfddf9d573ee1a09adc2396508735","abstract_canon_sha256":"5eb6a9951af71a57a02527f66b9aad7f0b602c027eb6ad1518cec17136d7cc2b"},"schema_version":"1.0"},"canonical_sha256":"c4c00b2ac89c11cda0fefcf377a2b10440c1cc5099a178249f39de6ea2d306e8","source":{"kind":"arxiv","id":"2309.08351","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.08351","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"arxiv_version","alias_value":"2309.08351v1","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.08351","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_12","alias_value":"YTAAWKWITQI4","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_16","alias_value":"YTAAWKWITQI43IH6","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_8","alias_value":"YTAAWKWI","created_at":"2026-07-05T06:51:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:YTAAWKWITQI43IH67TZXPIVRAR","target":"record","payload":{"canonical_record":{"source":{"id":"2309.08351","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-15T12:20:00Z","cross_cats_sorted":[],"title_canon_sha256":"3ab4e5ddc4d124b37a9f07d23b799844cccdfddf9d573ee1a09adc2396508735","abstract_canon_sha256":"5eb6a9951af71a57a02527f66b9aad7f0b602c027eb6ad1518cec17136d7cc2b"},"schema_version":"1.0"},"canonical_sha256":"c4c00b2ac89c11cda0fefcf377a2b10440c1cc5099a178249f39de6ea2d306e8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:51:05.509246Z","signature_b64":"256iSVpMziRakIRkcUc1W6UCWGS/2l/IYjVJ5g8P7hgASTVmTlaUauTCp7OKlYON2VGeVWGzVM99F5exaNFeDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c4c00b2ac89c11cda0fefcf377a2b10440c1cc5099a178249f39de6ea2d306e8","last_reissued_at":"2026-07-05T06:51:05.508858Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:51:05.508858Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.08351","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:51:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DpF9hT+i04w7J7fJ3XJQYDxxHJj61vVDl+icXNGPFfT9bm6UFHo1GX3fVVjK4rn7fPs7mL7ubpN0nvoY7m58AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:14:56.607315Z"},"content_sha256":"7a86772dcea8928f5edb37ad882ef344a869ec88f8774eb334c3010aeb90fb7e","schema_version":"1.0","event_id":"sha256:7a86772dcea8928f5edb37ad882ef344a869ec88f8774eb334c3010aeb90fb7e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:YTAAWKWITQI43IH67TZXPIVRAR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Headless Language Models: Learning without Predicting with Contrastive Weight Tying","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Beno\\^it Sagot, \\'Eric de la Clergerie, Nathan Godey","submitted_at":"2023-09-15T12:20:00Z","abstract_excerpt":"Self-supervised pre-training of language models usually consists in predicting probability distributions over extensive token vocabularies. In this study, we propose an innovative method that shifts away from probability prediction and instead focuses on reconstructing input embeddings in a contrastive fashion via Constrastive Weight Tying (CWT). We apply this approach to pretrain Headless Language Models in both monolingual and multilingual contexts. Our method offers practical advantages, substantially reducing training computational requirements by up to 20 times, while simultaneously enhan"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.08351","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.08351/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:51:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Gv7BsbdCcN1kyAxjWgUptUT+/Hn2XIa46wS3gCsCW+KJSoqW1ZqEdgkeCBgF6OGD7+0EkNE0hda/VzSwsUqYDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:14:56.607797Z"},"content_sha256":"327dae7a68d6ce942a3eea57115eb2ddabc0da004adc9ad7c763b89e17fe3509","schema_version":"1.0","event_id":"sha256:327dae7a68d6ce942a3eea57115eb2ddabc0da004adc9ad7c763b89e17fe3509"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YTAAWKWITQI43IH67TZXPIVRAR/bundle.json","state_url":"https://pith.science/pith/YTAAWKWITQI43IH67TZXPIVRAR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YTAAWKWITQI43IH67TZXPIVRAR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T13:14:56Z","links":{"resolver":"https://pith.science/pith/YTAAWKWITQI43IH67TZXPIVRAR","bundle":"https://pith.science/pith/YTAAWKWITQI43IH67TZXPIVRAR/bundle.json","state":"https://pith.science/pith/YTAAWKWITQI43IH67TZXPIVRAR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YTAAWKWITQI43IH67TZXPIVRAR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:YTAAWKWITQI43IH67TZXPIVRAR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5eb6a9951af71a57a02527f66b9aad7f0b602c027eb6ad1518cec17136d7cc2b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-15T12:20:00Z","title_canon_sha256":"3ab4e5ddc4d124b37a9f07d23b799844cccdfddf9d573ee1a09adc2396508735"},"schema_version":"1.0","source":{"id":"2309.08351","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.08351","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"arxiv_version","alias_value":"2309.08351v1","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.08351","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_12","alias_value":"YTAAWKWITQI4","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_16","alias_value":"YTAAWKWITQI43IH6","created_at":"2026-07-05T06:51:05Z"},{"alias_kind":"pith_short_8","alias_value":"YTAAWKWI","created_at":"2026-07-05T06:51:05Z"}],"graph_snapshots":[{"event_id":"sha256:327dae7a68d6ce942a3eea57115eb2ddabc0da004adc9ad7c763b89e17fe3509","target":"graph","created_at":"2026-07-05T06:51:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.08351/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Self-supervised pre-training of language models usually consists in predicting probability distributions over extensive token vocabularies. In this study, we propose an innovative method that shifts away from probability prediction and instead focuses on reconstructing input embeddings in a contrastive fashion via Constrastive Weight Tying (CWT). We apply this approach to pretrain Headless Language Models in both monolingual and multilingual contexts. Our method offers practical advantages, substantially reducing training computational requirements by up to 20 times, while simultaneously enhan","authors_text":"Beno\\^it Sagot, \\'Eric de la Clergerie, Nathan Godey","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-15T12:20:00Z","title":"Headless Language Models: Learning without Predicting with Contrastive Weight Tying"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.08351","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7a86772dcea8928f5edb37ad882ef344a869ec88f8774eb334c3010aeb90fb7e","target":"record","created_at":"2026-07-05T06:51:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5eb6a9951af71a57a02527f66b9aad7f0b602c027eb6ad1518cec17136d7cc2b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-15T12:20:00Z","title_canon_sha256":"3ab4e5ddc4d124b37a9f07d23b799844cccdfddf9d573ee1a09adc2396508735"},"schema_version":"1.0","source":{"id":"2309.08351","kind":"arxiv","version":1}},"canonical_sha256":"c4c00b2ac89c11cda0fefcf377a2b10440c1cc5099a178249f39de6ea2d306e8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c4c00b2ac89c11cda0fefcf377a2b10440c1cc5099a178249f39de6ea2d306e8","first_computed_at":"2026-07-05T06:51:05.508858Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:51:05.508858Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"256iSVpMziRakIRkcUc1W6UCWGS/2l/IYjVJ5g8P7hgASTVmTlaUauTCp7OKlYON2VGeVWGzVM99F5exaNFeDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:51:05.509246Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.08351","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7a86772dcea8928f5edb37ad882ef344a869ec88f8774eb334c3010aeb90fb7e","sha256:327dae7a68d6ce942a3eea57115eb2ddabc0da004adc9ad7c763b89e17fe3509"],"state_sha256":"71694fb0d18b3c94814e868b3923ce4423a75abadaaa966ecbc3536e2b61687e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QajuOZZNqzbw3y1gdZ/sDyGv/jtN1MA+e7xT8c3YuWAsSjjOg2tNw7lRFkVaSfWyVqFv0x9NLsOQ57KPxm+8Cg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T13:14:56.610945Z","bundle_sha256":"23dad4ff91b24ed88cc527b8d2f30f257b1284c7eb3d8f2c73379133191185af"}}