{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:RQNE67FKEODSQBU3HKFTNPABTX","short_pith_number":"pith:RQNE67FK","canonical_record":{"source":{"id":"2410.09605","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T17:50:58Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"f3e36defbccbf2aa063b4301ccbd8a8506d8f8a34d5fecca37573494ad80e957","abstract_canon_sha256":"f1e9c57cdd7035005ef3883dba816ff54f4ddd18fa08c65e42e2838082ca1f63"},"schema_version":"1.0"},"canonical_sha256":"8c1a4f7caa238728069b3a8b36bc019dd58ef580d2d78d5eefbde111ae89ef67","source":{"kind":"arxiv","id":"2410.09605","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.09605","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"arxiv_version","alias_value":"2410.09605v1","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.09605","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_12","alias_value":"RQNE67FKEODS","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_16","alias_value":"RQNE67FKEODSQBU3","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_8","alias_value":"RQNE67FK","created_at":"2026-07-05T09:20:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:RQNE67FKEODSQBU3HKFTNPABTX","target":"record","payload":{"canonical_record":{"source":{"id":"2410.09605","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T17:50:58Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"f3e36defbccbf2aa063b4301ccbd8a8506d8f8a34d5fecca37573494ad80e957","abstract_canon_sha256":"f1e9c57cdd7035005ef3883dba816ff54f4ddd18fa08c65e42e2838082ca1f63"},"schema_version":"1.0"},"canonical_sha256":"8c1a4f7caa238728069b3a8b36bc019dd58ef580d2d78d5eefbde111ae89ef67","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:20:06.063529Z","signature_b64":"2Bxy5wOgCp1BlOeCePZ1bEGnIe87lSqGGbxJkFAC5gwncWUydHvSSIOQZkS4ejjO0Y/+GOSwjXaUn0UUpWXgAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8c1a4f7caa238728069b3a8b36bc019dd58ef580d2d78d5eefbde111ae89ef67","last_reissued_at":"2026-07-05T09:20:06.063037Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:20:06.063037Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.09605","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:20:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qserRmunCNXtEmPdMgrbyHOJ5iYUwMFNI0y8yozBQV8E5DUlsmK8AifXs2dKhKDObNpjsKU6yK76xAPgrcZyCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T21:01:02.847105Z"},"content_sha256":"704c7930835187b552b8a8ebf04afab5f3d5905b3eff029b6aac1f1aea03a266","schema_version":"1.0","event_id":"sha256:704c7930835187b552b8a8ebf04afab5f3d5905b3eff029b6aac1f1aea03a266"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:RQNE67FKEODSQBU3HKFTNPABTX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Training Dynamics of Transformers to Recognize Word Co-occurrence via Gradient Flow Analysis","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Bhavya Kailkhura, Hongru Yang, Yingbin Liang, Zhangyang Wang","submitted_at":"2024-10-12T17:50:58Z","abstract_excerpt":"Understanding the training dynamics of transformers is important to explain the impressive capabilities behind large language models. In this work, we study the dynamics of training a shallow transformer on a task of recognizing co-occurrence of two designated words. In the literature of studying training dynamics of transformers, several simplifications are commonly adopted such as weight reparameterization, attention linearization, special initialization, and lazy regime. In contrast, we analyze the gradient flow dynamics of simultaneously training three attention matrices and a linear MLP l"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.09605","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.09605/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:20:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JaGalUW1RvR5IsQJUo0wwO+NC614aTkxoqHWWD51AJ+LzMIe8GCXJy6A6vG1t5pg8k3ZfKsjlcCTamvn8972AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T21:01:02.847603Z"},"content_sha256":"876b69bfebeb0a8896bd1c42333743d915e01eeb513268ac838303dd0b62188a","schema_version":"1.0","event_id":"sha256:876b69bfebeb0a8896bd1c42333743d915e01eeb513268ac838303dd0b62188a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RQNE67FKEODSQBU3HKFTNPABTX/bundle.json","state_url":"https://pith.science/pith/RQNE67FKEODSQBU3HKFTNPABTX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RQNE67FKEODSQBU3HKFTNPABTX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T21:01:02Z","links":{"resolver":"https://pith.science/pith/RQNE67FKEODSQBU3HKFTNPABTX","bundle":"https://pith.science/pith/RQNE67FKEODSQBU3HKFTNPABTX/bundle.json","state":"https://pith.science/pith/RQNE67FKEODSQBU3HKFTNPABTX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RQNE67FKEODSQBU3HKFTNPABTX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:RQNE67FKEODSQBU3HKFTNPABTX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f1e9c57cdd7035005ef3883dba816ff54f4ddd18fa08c65e42e2838082ca1f63","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T17:50:58Z","title_canon_sha256":"f3e36defbccbf2aa063b4301ccbd8a8506d8f8a34d5fecca37573494ad80e957"},"schema_version":"1.0","source":{"id":"2410.09605","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.09605","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"arxiv_version","alias_value":"2410.09605v1","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.09605","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_12","alias_value":"RQNE67FKEODS","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_16","alias_value":"RQNE67FKEODSQBU3","created_at":"2026-07-05T09:20:06Z"},{"alias_kind":"pith_short_8","alias_value":"RQNE67FK","created_at":"2026-07-05T09:20:06Z"}],"graph_snapshots":[{"event_id":"sha256:876b69bfebeb0a8896bd1c42333743d915e01eeb513268ac838303dd0b62188a","target":"graph","created_at":"2026-07-05T09:20:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.09605/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Understanding the training dynamics of transformers is important to explain the impressive capabilities behind large language models. In this work, we study the dynamics of training a shallow transformer on a task of recognizing co-occurrence of two designated words. In the literature of studying training dynamics of transformers, several simplifications are commonly adopted such as weight reparameterization, attention linearization, special initialization, and lazy regime. In contrast, we analyze the gradient flow dynamics of simultaneously training three attention matrices and a linear MLP l","authors_text":"Bhavya Kailkhura, Hongru Yang, Yingbin Liang, Zhangyang Wang","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T17:50:58Z","title":"Training Dynamics of Transformers to Recognize Word Co-occurrence via Gradient Flow Analysis"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.09605","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:704c7930835187b552b8a8ebf04afab5f3d5905b3eff029b6aac1f1aea03a266","target":"record","created_at":"2026-07-05T09:20:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f1e9c57cdd7035005ef3883dba816ff54f4ddd18fa08c65e42e2838082ca1f63","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T17:50:58Z","title_canon_sha256":"f3e36defbccbf2aa063b4301ccbd8a8506d8f8a34d5fecca37573494ad80e957"},"schema_version":"1.0","source":{"id":"2410.09605","kind":"arxiv","version":1}},"canonical_sha256":"8c1a4f7caa238728069b3a8b36bc019dd58ef580d2d78d5eefbde111ae89ef67","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8c1a4f7caa238728069b3a8b36bc019dd58ef580d2d78d5eefbde111ae89ef67","first_computed_at":"2026-07-05T09:20:06.063037Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:20:06.063037Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2Bxy5wOgCp1BlOeCePZ1bEGnIe87lSqGGbxJkFAC5gwncWUydHvSSIOQZkS4ejjO0Y/+GOSwjXaUn0UUpWXgAw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:20:06.063529Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.09605","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:704c7930835187b552b8a8ebf04afab5f3d5905b3eff029b6aac1f1aea03a266","sha256:876b69bfebeb0a8896bd1c42333743d915e01eeb513268ac838303dd0b62188a"],"state_sha256":"90f120aa92198f9fe744c2a760b6b3838656ec67587a84493179feeab4924bca"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pCGTjRavx7+imgthf12/7iSFyqAjJMDg2kuVZXXEghFkrHfl4sIBdzqFTKY+4+Io8djU8GuLtATHM5Z5qqf1Aw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T21:01:02.851273Z","bundle_sha256":"31c148a08ef53be54c3b5c6059f600c50554663a95924546828e433f3a87740a"}}