{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:N3NUAWMNULWQ2IPBGILGGZISXG","short_pith_number":"pith:N3NUAWMN","canonical_record":{"source":{"id":"2309.07311","kind":"arxiv","version":6},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-13T20:57:11Z","cross_cats_sorted":[],"title_canon_sha256":"a06332f862f8d277f46b32240dd7b615f7b4df9fc561c36e349d930a3a51fa2e","abstract_canon_sha256":"6346c59b99c3f37a73c3758623fca7d7ad06e127bbbe76d6530b9ee5d1e09fd8"},"schema_version":"1.0"},"canonical_sha256":"6edb40598da2ed0d21e13216636512b9bb558ccad1984fdb69060c9117538995","source":{"kind":"arxiv","id":"2309.07311","version":6},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.07311","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"arxiv_version","alias_value":"2309.07311v6","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.07311","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_12","alias_value":"N3NUAWMNULWQ","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_16","alias_value":"N3NUAWMNULWQ2IPB","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_8","alias_value":"N3NUAWMN","created_at":"2026-07-05T10:37:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:N3NUAWMNULWQ2IPBGILGGZISXG","target":"record","payload":{"canonical_record":{"source":{"id":"2309.07311","kind":"arxiv","version":6},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-13T20:57:11Z","cross_cats_sorted":[],"title_canon_sha256":"a06332f862f8d277f46b32240dd7b615f7b4df9fc561c36e349d930a3a51fa2e","abstract_canon_sha256":"6346c59b99c3f37a73c3758623fca7d7ad06e127bbbe76d6530b9ee5d1e09fd8"},"schema_version":"1.0"},"canonical_sha256":"6edb40598da2ed0d21e13216636512b9bb558ccad1984fdb69060c9117538995","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:37:05.103803Z","signature_b64":"AV9FkymIqhNcxqls4aEGsLhXWR8urs8NXgU+MtJQW7jfeUBO8kUsULPaBCdO9aF9hM5J0esVN/pFNKOSBmoMDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6edb40598da2ed0d21e13216636512b9bb558ccad1984fdb69060c9117538995","last_reissued_at":"2026-07-05T10:37:05.102890Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:37:05.102890Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.07311","source_version":6,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:37:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dmgPYOqtHBJleepb3WG9ptnHG/+7YrnCWXH2jd/NvVkykmHwnHyT6U4auuBqtK/9ACRDHKPS69M4O3cqXKksCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-12T23:32:57.345125Z"},"content_sha256":"63ab58f760a836a164367660c9b600de48c7a5c5b866e992495b24e126277cd7","schema_version":"1.0","event_id":"sha256:63ab58f760a836a164367660c9b600de48c7a5c5b866e992495b24e126277cd7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:N3NUAWMNULWQ2IPBGILGGZISXG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Sudden Drops in the Loss: Syntax Acquisition, Phase Transitions, and Simplicity Bias in MLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Angelica Chen, Kyunghyun Cho, Matthew L. Leavitt, Naomi Saphra, Ravid Shwartz-Ziv","submitted_at":"2023-09-13T20:57:11Z","abstract_excerpt":"Most interpretability research in NLP focuses on understanding the behavior and features of a fully trained model. However, certain insights into model behavior may only be accessible by observing the trajectory of the training process. We present a case study of syntax acquisition in masked language models (MLMs) that demonstrates how analyzing the evolution of interpretable artifacts throughout training deepens our understanding of emergent behavior. In particular, we study Syntactic Attention Structure (SAS), a naturally emerging property of MLMs wherein specific Transformer heads tend to f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.07311","kind":"arxiv","version":6},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.07311/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:37:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HHWQ5yFieZPDZfigpuLhXd33Ue9b3nEXmO7RgKdAQsNV7XqksHQF8S4SafMv2ciUpYp7Gd8Jsgl1jtoDRQx/AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-12T23:32:57.345527Z"},"content_sha256":"d3b19c9ea5e0ec5d394c793d270dbfda53fa6a274f9805969c622ee8b1e13621","schema_version":"1.0","event_id":"sha256:d3b19c9ea5e0ec5d394c793d270dbfda53fa6a274f9805969c622ee8b1e13621"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/N3NUAWMNULWQ2IPBGILGGZISXG/bundle.json","state_url":"https://pith.science/pith/N3NUAWMNULWQ2IPBGILGGZISXG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/N3NUAWMNULWQ2IPBGILGGZISXG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-12T23:32:57Z","links":{"resolver":"https://pith.science/pith/N3NUAWMNULWQ2IPBGILGGZISXG","bundle":"https://pith.science/pith/N3NUAWMNULWQ2IPBGILGGZISXG/bundle.json","state":"https://pith.science/pith/N3NUAWMNULWQ2IPBGILGGZISXG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/N3NUAWMNULWQ2IPBGILGGZISXG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:N3NUAWMNULWQ2IPBGILGGZISXG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6346c59b99c3f37a73c3758623fca7d7ad06e127bbbe76d6530b9ee5d1e09fd8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-13T20:57:11Z","title_canon_sha256":"a06332f862f8d277f46b32240dd7b615f7b4df9fc561c36e349d930a3a51fa2e"},"schema_version":"1.0","source":{"id":"2309.07311","kind":"arxiv","version":6}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.07311","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"arxiv_version","alias_value":"2309.07311v6","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.07311","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_12","alias_value":"N3NUAWMNULWQ","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_16","alias_value":"N3NUAWMNULWQ2IPB","created_at":"2026-07-05T10:37:05Z"},{"alias_kind":"pith_short_8","alias_value":"N3NUAWMN","created_at":"2026-07-05T10:37:05Z"}],"graph_snapshots":[{"event_id":"sha256:d3b19c9ea5e0ec5d394c793d270dbfda53fa6a274f9805969c622ee8b1e13621","target":"graph","created_at":"2026-07-05T10:37:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.07311/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most interpretability research in NLP focuses on understanding the behavior and features of a fully trained model. However, certain insights into model behavior may only be accessible by observing the trajectory of the training process. We present a case study of syntax acquisition in masked language models (MLMs) that demonstrates how analyzing the evolution of interpretable artifacts throughout training deepens our understanding of emergent behavior. In particular, we study Syntactic Attention Structure (SAS), a naturally emerging property of MLMs wherein specific Transformer heads tend to f","authors_text":"Angelica Chen, Kyunghyun Cho, Matthew L. Leavitt, Naomi Saphra, Ravid Shwartz-Ziv","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-13T20:57:11Z","title":"Sudden Drops in the Loss: Syntax Acquisition, Phase Transitions, and Simplicity Bias in MLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.07311","kind":"arxiv","version":6},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:63ab58f760a836a164367660c9b600de48c7a5c5b866e992495b24e126277cd7","target":"record","created_at":"2026-07-05T10:37:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6346c59b99c3f37a73c3758623fca7d7ad06e127bbbe76d6530b9ee5d1e09fd8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-13T20:57:11Z","title_canon_sha256":"a06332f862f8d277f46b32240dd7b615f7b4df9fc561c36e349d930a3a51fa2e"},"schema_version":"1.0","source":{"id":"2309.07311","kind":"arxiv","version":6}},"canonical_sha256":"6edb40598da2ed0d21e13216636512b9bb558ccad1984fdb69060c9117538995","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6edb40598da2ed0d21e13216636512b9bb558ccad1984fdb69060c9117538995","first_computed_at":"2026-07-05T10:37:05.102890Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:37:05.102890Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AV9FkymIqhNcxqls4aEGsLhXWR8urs8NXgU+MtJQW7jfeUBO8kUsULPaBCdO9aF9hM5J0esVN/pFNKOSBmoMDg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:37:05.103803Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.07311","source_kind":"arxiv","source_version":6}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:63ab58f760a836a164367660c9b600de48c7a5c5b866e992495b24e126277cd7","sha256:d3b19c9ea5e0ec5d394c793d270dbfda53fa6a274f9805969c622ee8b1e13621"],"state_sha256":"fb2aa14049982f268e883d5848d2891b6e3a5afa1617e81eaa3415110f6ee1ca"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7kbQ8WRzPArqzEuZlqtsFVyku3n52Jr+Pb5YQ9GA5s7FaUkVEXxOtEHWhH63oVhD1c0d/JcPv/zP1ZMwd+jUCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-12T23:32:57.348122Z","bundle_sha256":"7891d5cdc78534feb247fa576a7ba44dd25711445dbaf246d3ad0b3dbfa391a7"}}