{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:43NMRLZKWKLTAW44GHYX27HY4T","short_pith_number":"pith:43NMRLZK","canonical_record":{"source":{"id":"2410.24050","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:46:10Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"a2135071acbcab095959a83b7537b4ebf9534369c36f34bced02b634483704e9","abstract_canon_sha256":"34d575ea2e50906b5ac7d307ce6b4d580d9522635d9df3916cd2325662ccae5a"},"schema_version":"1.0"},"canonical_sha256":"e6dac8af2ab297305b9c31f17d7cf8e4d69395a705037cc1e6b19af4f8f8b596","source":{"kind":"arxiv","id":"2410.24050","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.24050","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"arxiv_version","alias_value":"2410.24050v3","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.24050","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_12","alias_value":"43NMRLZKWKLT","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_16","alias_value":"43NMRLZKWKLTAW44","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_8","alias_value":"43NMRLZK","created_at":"2026-06-30T02:18:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:43NMRLZKWKLTAW44GHYX27HY4T","target":"record","payload":{"canonical_record":{"source":{"id":"2410.24050","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:46:10Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"a2135071acbcab095959a83b7537b4ebf9534369c36f34bced02b634483704e9","abstract_canon_sha256":"34d575ea2e50906b5ac7d307ce6b4d580d9522635d9df3916cd2325662ccae5a"},"schema_version":"1.0"},"canonical_sha256":"e6dac8af2ab297305b9c31f17d7cf8e4d69395a705037cc1e6b19af4f8f8b596","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-30T02:18:03.558818Z","signature_b64":"5lTXDTYLsnbRKfM8WR15heywXBm0osG9hEJn3YGMsutZygqsFG/jiWd1/PE32+utwIL9bMvyEX7WJ9nG/8YcAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e6dac8af2ab297305b9c31f17d7cf8e4d69395a705037cc1e6b19af4f8f8b596","last_reissued_at":"2026-06-30T02:18:03.558145Z","signature_status":"signed_v1","first_computed_at":"2026-06-30T02:18:03.558145Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.24050","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-30T02:18:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7fTcqjTDoxH9GOkyEC8QfOByTOtbWvP02LzVxMRjoVcTvKLc8L+dfgTBUTWIiIxLPH3mr/ywC5M7jcnnJ4JwBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T01:07:11.890992Z"},"content_sha256":"5c82494dc55d7ac136d442d3066d9d21184248de6fdb05e5297500780a98360f","schema_version":"1.0","event_id":"sha256:5c82494dc55d7ac136d442d3066d9d21184248de6fdb05e5297500780a98360f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:43NMRLZKWKLTAW44GHYX27HY4T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Mechanistic Study of Transformers Training Dynamics","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Ambroise Odonnat, Vivien Cabannes, Wassim Bouaziz","submitted_at":"2024-10-31T15:46:10Z","abstract_excerpt":"Large-scale pretraining of transformers has been central to the success of foundation models. However, the scale of those models limits our understanding of the mechanisms at play during optimization. In this work, we study the training dynamics of transformers in a controlled and interpretable setting. On the sparse modular addition task, we demonstrate that specialized attention circuits, called clustering heads, can be implemented during gradient descent to solve the problem. Our experiments show that such pathways naturally emerge during training. By monitoring the evolution of tokens via "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.24050","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.24050/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-30T02:18:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZriUW8Cyv7z5ghWQTCCNt0n2mujUvWcHnw2UlxVhOwobSRnLT7Mol7+d+/tPiQPyRohSSsOQcWvyi5UXa2LvAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T01:07:11.891336Z"},"content_sha256":"aa51a10f2fd7cd9dc6d5591c6f96cb9a264dbda105d1cf9ee2d66c8bbdc96065","schema_version":"1.0","event_id":"sha256:aa51a10f2fd7cd9dc6d5591c6f96cb9a264dbda105d1cf9ee2d66c8bbdc96065"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/43NMRLZKWKLTAW44GHYX27HY4T/bundle.json","state_url":"https://pith.science/pith/43NMRLZKWKLTAW44GHYX27HY4T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/43NMRLZKWKLTAW44GHYX27HY4T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-19T01:07:11Z","links":{"resolver":"https://pith.science/pith/43NMRLZKWKLTAW44GHYX27HY4T","bundle":"https://pith.science/pith/43NMRLZKWKLTAW44GHYX27HY4T/bundle.json","state":"https://pith.science/pith/43NMRLZKWKLTAW44GHYX27HY4T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/43NMRLZKWKLTAW44GHYX27HY4T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:43NMRLZKWKLTAW44GHYX27HY4T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34d575ea2e50906b5ac7d307ce6b4d580d9522635d9df3916cd2325662ccae5a","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:46:10Z","title_canon_sha256":"a2135071acbcab095959a83b7537b4ebf9534369c36f34bced02b634483704e9"},"schema_version":"1.0","source":{"id":"2410.24050","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.24050","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"arxiv_version","alias_value":"2410.24050v3","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.24050","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_12","alias_value":"43NMRLZKWKLT","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_16","alias_value":"43NMRLZKWKLTAW44","created_at":"2026-06-30T02:18:03Z"},{"alias_kind":"pith_short_8","alias_value":"43NMRLZK","created_at":"2026-06-30T02:18:03Z"}],"graph_snapshots":[{"event_id":"sha256:aa51a10f2fd7cd9dc6d5591c6f96cb9a264dbda105d1cf9ee2d66c8bbdc96065","target":"graph","created_at":"2026-06-30T02:18:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.24050/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large-scale pretraining of transformers has been central to the success of foundation models. However, the scale of those models limits our understanding of the mechanisms at play during optimization. In this work, we study the training dynamics of transformers in a controlled and interpretable setting. On the sparse modular addition task, we demonstrate that specialized attention circuits, called clustering heads, can be implemented during gradient descent to solve the problem. Our experiments show that such pathways naturally emerge during training. By monitoring the evolution of tokens via ","authors_text":"Ambroise Odonnat, Vivien Cabannes, Wassim Bouaziz","cross_cats":["stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:46:10Z","title":"A Mechanistic Study of Transformers Training Dynamics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.24050","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5c82494dc55d7ac136d442d3066d9d21184248de6fdb05e5297500780a98360f","target":"record","created_at":"2026-06-30T02:18:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34d575ea2e50906b5ac7d307ce6b4d580d9522635d9df3916cd2325662ccae5a","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:46:10Z","title_canon_sha256":"a2135071acbcab095959a83b7537b4ebf9534369c36f34bced02b634483704e9"},"schema_version":"1.0","source":{"id":"2410.24050","kind":"arxiv","version":3}},"canonical_sha256":"e6dac8af2ab297305b9c31f17d7cf8e4d69395a705037cc1e6b19af4f8f8b596","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e6dac8af2ab297305b9c31f17d7cf8e4d69395a705037cc1e6b19af4f8f8b596","first_computed_at":"2026-06-30T02:18:03.558145Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-06-30T02:18:03.558145Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"5lTXDTYLsnbRKfM8WR15heywXBm0osG9hEJn3YGMsutZygqsFG/jiWd1/PE32+utwIL9bMvyEX7WJ9nG/8YcAA==","signature_status":"signed_v1","signed_at":"2026-06-30T02:18:03.558818Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.24050","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5c82494dc55d7ac136d442d3066d9d21184248de6fdb05e5297500780a98360f","sha256:aa51a10f2fd7cd9dc6d5591c6f96cb9a264dbda105d1cf9ee2d66c8bbdc96065"],"state_sha256":"b09eaded439d19c3bca7fc0f26ff4fe2c5e5f3b39d4ded4d9554ec47e1feedeb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gquQMCraPwKiX21zH8/OtvHPMspFd+SZsqKlQ9Lv0WbcwqSGUl6kuhAeoe9tCUWqcUnimYE7iMd0xyJiDuNHCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-19T01:07:11.894996Z","bundle_sha256":"228258adf49d5fe1e0a4e0c0eb18a70698fc2510a41076040f29874a5b25b9bb"}}