{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YL2YPG66MYVTV7SID7VJBFZNFZ","short_pith_number":"pith:YL2YPG66","canonical_record":{"source":{"id":"2404.07647","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-11T11:10:36Z","cross_cats_sorted":[],"title_canon_sha256":"def5209b67fdb90d8bd00ff2204abdc3cbe36cb9a627c126ff72bdbb0e97ee8e","abstract_canon_sha256":"41c42a14c5f378f3eafe25a399fb272cdb061bb3a6b68f129c630aa1e1d8978f"},"schema_version":"1.0"},"canonical_sha256":"c2f5879bde662b3afe481fea90972d2e63c00b013dbfffddf683c69393efd741","source":{"kind":"arxiv","id":"2404.07647","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.07647","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"arxiv_version","alias_value":"2404.07647v1","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.07647","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_12","alias_value":"YL2YPG66MYVT","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_16","alias_value":"YL2YPG66MYVTV7SI","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_8","alias_value":"YL2YPG66","created_at":"2026-07-05T08:06:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YL2YPG66MYVTV7SID7VJBFZNFZ","target":"record","payload":{"canonical_record":{"source":{"id":"2404.07647","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-11T11:10:36Z","cross_cats_sorted":[],"title_canon_sha256":"def5209b67fdb90d8bd00ff2204abdc3cbe36cb9a627c126ff72bdbb0e97ee8e","abstract_canon_sha256":"41c42a14c5f378f3eafe25a399fb272cdb061bb3a6b68f129c630aa1e1d8978f"},"schema_version":"1.0"},"canonical_sha256":"c2f5879bde662b3afe481fea90972d2e63c00b013dbfffddf683c69393efd741","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:06:55.732880Z","signature_b64":"V5n+gM5Ddq++mnfoKBYX/8OwtBG1wjDRN/hyeM69rme1T8PiNSYBSLnFiMtqarQ7BshzJ7OhFSYNi4p3j8skDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c2f5879bde662b3afe481fea90972d2e63c00b013dbfffddf683c69393efd741","last_reissued_at":"2026-07-05T08:06:55.732394Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:06:55.732394Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.07647","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:06:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fA0OIi8RqI2Jq/A7UHlMGkiMDHJuw93GQKXKFV0MLIxfvh1WB8jnm1DRxe1UugXLzCrGblLYPKGROI0XFygZDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T08:45:36.378903Z"},"content_sha256":"203049b367784e16a768d58719601f0d36518f79265f109073f1b404055fcc15","schema_version":"1.0","event_id":"sha256:203049b367784e16a768d58719601f0d36518f79265f109073f1b404055fcc15"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YL2YPG66MYVTV7SID7VJBFZNFZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Why do small language models underperform? Studying Language Model Saturation via the Softmax Bottleneck","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Beno\\^it Sagot, \\'Eric de la Clergerie, Nathan Godey","submitted_at":"2024-04-11T11:10:36Z","abstract_excerpt":"Recent advances in language modeling consist in pretraining highly parameterized neural networks on extremely large web-mined text corpora. Training and inference with such models can be costly in practice, which incentivizes the use of smaller counterparts. However, it has been observed that smaller models can suffer from saturation, characterized as a drop in performance at some advanced point in training followed by a plateau. In this paper, we find that such saturation can be explained by a mismatch between the hidden dimension of smaller models and the high rank of the target contextual p"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.07647","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.07647/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:06:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oKLfH0H417iL+2dGSRJURd1jyvL4Xm9r/JxwjjE+qbH8AqNkN5uL77wtkP++IlHqI65A45PBlBlM+y+KSa5LAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T08:45:36.379944Z"},"content_sha256":"6c3c34716f339042eae5e47278f81b6649d6361be898dfe2945387801b84ab63","schema_version":"1.0","event_id":"sha256:6c3c34716f339042eae5e47278f81b6649d6361be898dfe2945387801b84ab63"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/bundle.json","state_url":"https://pith.science/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T08:45:36Z","links":{"resolver":"https://pith.science/pith/YL2YPG66MYVTV7SID7VJBFZNFZ","bundle":"https://pith.science/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/bundle.json","state":"https://pith.science/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YL2YPG66MYVTV7SID7VJBFZNFZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YL2YPG66MYVTV7SID7VJBFZNFZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"41c42a14c5f378f3eafe25a399fb272cdb061bb3a6b68f129c630aa1e1d8978f","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-11T11:10:36Z","title_canon_sha256":"def5209b67fdb90d8bd00ff2204abdc3cbe36cb9a627c126ff72bdbb0e97ee8e"},"schema_version":"1.0","source":{"id":"2404.07647","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.07647","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"arxiv_version","alias_value":"2404.07647v1","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.07647","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_12","alias_value":"YL2YPG66MYVT","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_16","alias_value":"YL2YPG66MYVTV7SI","created_at":"2026-07-05T08:06:55Z"},{"alias_kind":"pith_short_8","alias_value":"YL2YPG66","created_at":"2026-07-05T08:06:55Z"}],"graph_snapshots":[{"event_id":"sha256:6c3c34716f339042eae5e47278f81b6649d6361be898dfe2945387801b84ab63","target":"graph","created_at":"2026-07-05T08:06:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.07647/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advances in language modeling consist in pretraining highly parameterized neural networks on extremely large web-mined text corpora. Training and inference with such models can be costly in practice, which incentivizes the use of smaller counterparts. However, it has been observed that smaller models can suffer from saturation, characterized as a drop in performance at some advanced point in training followed by a plateau. In this paper, we find that such saturation can be explained by a mismatch between the hidden dimension of smaller models and the high rank of the target contextual p","authors_text":"Beno\\^it Sagot, \\'Eric de la Clergerie, Nathan Godey","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-11T11:10:36Z","title":"Why do small language models underperform? Studying Language Model Saturation via the Softmax Bottleneck"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.07647","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:203049b367784e16a768d58719601f0d36518f79265f109073f1b404055fcc15","target":"record","created_at":"2026-07-05T08:06:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"41c42a14c5f378f3eafe25a399fb272cdb061bb3a6b68f129c630aa1e1d8978f","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-11T11:10:36Z","title_canon_sha256":"def5209b67fdb90d8bd00ff2204abdc3cbe36cb9a627c126ff72bdbb0e97ee8e"},"schema_version":"1.0","source":{"id":"2404.07647","kind":"arxiv","version":1}},"canonical_sha256":"c2f5879bde662b3afe481fea90972d2e63c00b013dbfffddf683c69393efd741","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c2f5879bde662b3afe481fea90972d2e63c00b013dbfffddf683c69393efd741","first_computed_at":"2026-07-05T08:06:55.732394Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:06:55.732394Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"V5n+gM5Ddq++mnfoKBYX/8OwtBG1wjDRN/hyeM69rme1T8PiNSYBSLnFiMtqarQ7BshzJ7OhFSYNi4p3j8skDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:06:55.732880Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.07647","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:203049b367784e16a768d58719601f0d36518f79265f109073f1b404055fcc15","sha256:6c3c34716f339042eae5e47278f81b6649d6361be898dfe2945387801b84ab63"],"state_sha256":"8d3cbb69a3594e5f9a1fdc8a9093029024af920ade01d71656bf4fff2f3d30ce"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nA1xSAJiVDhqbXJ1CKDxmA9Nlij/x8bYHnVuXgrCij/WPGsgJ5V8sj4nfwkyAzxzSVGVuGCtukRfyRgm5Pr/Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T08:45:36.388046Z","bundle_sha256":"40287368c7d4d4f3d4201b98fc1de1fd4835006f23967db08289cd9d6098e774"}}