{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:35IEMCHVXCUSHAY44RTFQU7W42","short_pith_number":"pith:35IEMCHV","schema_version":"1.0","canonical_sha256":"df504608f5b8a923831ce4665853f6e687c2f2530bdbe19388e64fb9d66522f1","source":{"kind":"arxiv","id":"2311.00863","version":1},"attestation_state":"computed","paper":{"title":"Training Dynamics of Contextual N-Grams in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Lovis Heindrich, Lucia Quirke, Neel Nanda, Wes Gurnee","submitted_at":"2023-11-01T21:32:51Z","abstract_excerpt":"Prior work has shown the existence of contextual neurons in language models, including a neuron that activates on German text. We show that this neuron exists within a broader contextual n-gram circuit: we find late layer neurons which recognize and continue n-grams common in German text, but which only activate if the German neuron is active. We investigate the formation of this circuit throughout training and find that it is an example of what we call a second-order circuit. In particular, both the constituent n-gram circuits and the German detection circuit which culminates in the German ne"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2311.00863","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-01T21:32:51Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"fdff694e45272d3b80c8999b3e3c4d0edb015b9f364f4fa7cec263d6484d2277","abstract_canon_sha256":"1a0e14bc0d8beaebec2b42414b820d95929c2a0f1474f944046e5020e01b2b51"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:08:16.724015Z","signature_b64":"z/fpxxaFaOIT0ZehLc2IYsOuxXCXT4Rh7bAV4YDrI7Y15RAgUtU6KdNeZYXY6vmaWWORZeWQ8Qrb1ba7laaXAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"df504608f5b8a923831ce4665853f6e687c2f2530bdbe19388e64fb9d66522f1","last_reissued_at":"2026-07-05T07:08:16.723528Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:08:16.723528Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Training Dynamics of Contextual N-Grams in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Lovis Heindrich, Lucia Quirke, Neel Nanda, Wes Gurnee","submitted_at":"2023-11-01T21:32:51Z","abstract_excerpt":"Prior work has shown the existence of contextual neurons in language models, including a neuron that activates on German text. We show that this neuron exists within a broader contextual n-gram circuit: we find late layer neurons which recognize and continue n-grams common in German text, but which only activate if the German neuron is active. We investigate the formation of this circuit throughout training and find that it is an example of what we call a second-order circuit. In particular, both the constituent n-gram circuits and the German detection circuit which culminates in the German ne"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.00863","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.00863/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2311.00863","created_at":"2026-07-05T07:08:16.723588+00:00"},{"alias_kind":"arxiv_version","alias_value":"2311.00863v1","created_at":"2026-07-05T07:08:16.723588+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.00863","created_at":"2026-07-05T07:08:16.723588+00:00"},{"alias_kind":"pith_short_12","alias_value":"35IEMCHVXCUS","created_at":"2026-07-05T07:08:16.723588+00:00"},{"alias_kind":"pith_short_16","alias_value":"35IEMCHVXCUSHAY4","created_at":"2026-07-05T07:08:16.723588+00:00"},{"alias_kind":"pith_short_8","alias_value":"35IEMCHV","created_at":"2026-07-05T07:08:16.723588+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42","json":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42.json","graph_json":"https://pith.science/api/pith-number/35IEMCHVXCUSHAY44RTFQU7W42/graph.json","events_json":"https://pith.science/api/pith-number/35IEMCHVXCUSHAY44RTFQU7W42/events.json","paper":"https://pith.science/paper/35IEMCHV"},"agent_actions":{"view_html":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42","download_json":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42.json","view_paper":"https://pith.science/paper/35IEMCHV","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2311.00863&json=true","fetch_graph":"https://pith.science/api/pith-number/35IEMCHVXCUSHAY44RTFQU7W42/graph.json","fetch_events":"https://pith.science/api/pith-number/35IEMCHVXCUSHAY44RTFQU7W42/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42/action/timestamp_anchor","attest_storage":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42/action/storage_attestation","attest_author":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42/action/author_attestation","sign_citation":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42/action/citation_signature","submit_replication":"https://pith.science/pith/35IEMCHVXCUSHAY44RTFQU7W42/action/replication_record"}},"created_at":"2026-07-05T07:08:16.723588+00:00","updated_at":"2026-07-05T07:08:16.723588+00:00"}