{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2WHKOAPRDIVFGJYI3ZZ27H7EWT","short_pith_number":"pith:2WHKOAPR","canonical_record":{"source":{"id":"2405.12250","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-19T22:44:00Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"52afdca733a8dfa362cebcd2dac952722fa4bf4ceb7641e8fb2e5466b1a7cf55","abstract_canon_sha256":"edf8b7debda7d871bede1282a9bae4f80973ea9a32d7be9ce743d87d063ed0ab"},"schema_version":"1.0"},"canonical_sha256":"d58ea701f11a2a532708de73af9fe4b4e45fac258a4bc422ae5e7ce6b77bc49c","source":{"kind":"arxiv","id":"2405.12250","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.12250","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"arxiv_version","alias_value":"2405.12250v1","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.12250","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_12","alias_value":"2WHKOAPRDIVF","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_16","alias_value":"2WHKOAPRDIVFGJYI","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_8","alias_value":"2WHKOAPR","created_at":"2026-07-05T08:21:17Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2WHKOAPRDIVFGJYI3ZZ27H7EWT","target":"record","payload":{"canonical_record":{"source":{"id":"2405.12250","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-19T22:44:00Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"52afdca733a8dfa362cebcd2dac952722fa4bf4ceb7641e8fb2e5466b1a7cf55","abstract_canon_sha256":"edf8b7debda7d871bede1282a9bae4f80973ea9a32d7be9ce743d87d063ed0ab"},"schema_version":"1.0"},"canonical_sha256":"d58ea701f11a2a532708de73af9fe4b4e45fac258a4bc422ae5e7ce6b77bc49c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:21:17.505298Z","signature_b64":"pq8YgQZCeV+R7kr8lVhiIbKWVYb0OxFsTE85m0LSSdricRCJlyzX8ume6x0XuI21419fUk+wpxA7vwiDEXVmCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d58ea701f11a2a532708de73af9fe4b4e45fac258a4bc422ae5e7ce6b77bc49c","last_reissued_at":"2026-07-05T08:21:17.504867Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:21:17.504867Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.12250","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:21:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DMjNCz7si8fALr4De1N7S8lG9ZgobUf9xxsRbVLWAOxmscx6VNg7DyhltRPp4cOzczDISvN97WV5d1+7Vy9GAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T23:58:15.983189Z"},"content_sha256":"06a8617dd51baa712e5ec298df3ee8a83f1bc96ad7b2591109aae9bf3b7b0a42","schema_version":"1.0","event_id":"sha256:06a8617dd51baa712e5ec298df3ee8a83f1bc96ad7b2591109aae9bf3b7b0a42"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2WHKOAPRDIVFGJYI3ZZ27H7EWT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Your Transformer is Secretly Linear","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Andrey Kuznetsov, Anton Razzhigaev, Denis Dimitrov, Elizaveta Goncharova, Ivan Oseledets, Matvey Mikhalchuk, Nikolai Gerasimenko","submitted_at":"2024-05-19T22:44:00Z","abstract_excerpt":"This paper reveals a novel linear characteristic exclusive to transformer decoders, including models such as GPT, LLaMA, OPT, BLOOM and others. We analyze embedding transformations between sequential layers, uncovering a near-perfect linear relationship (Procrustes similarity score of 0.99). However, linearity decreases when the residual component is removed due to a consistently low output norm of the transformer layer. Our experiments show that removing or linearly approximating some of the most linear blocks of transformers does not affect significantly the loss or model performance. Moreov"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.12250","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.12250/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:21:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jfa9Np4xOOrxZocA1hApc3Cxd4+/BBu9J1LFL269OAVkz4YoufYxnwf/KzQs2yYgeyJVvKU1hI+5xgKwyLQHAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T23:58:15.983580Z"},"content_sha256":"f00d116536e8e9132f48329e203b76c8135a20a5a88e5424f1c5827c2b54b589","schema_version":"1.0","event_id":"sha256:f00d116536e8e9132f48329e203b76c8135a20a5a88e5424f1c5827c2b54b589"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/bundle.json","state_url":"https://pith.science/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-16T23:58:15Z","links":{"resolver":"https://pith.science/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT","bundle":"https://pith.science/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/bundle.json","state":"https://pith.science/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2WHKOAPRDIVFGJYI3ZZ27H7EWT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2WHKOAPRDIVFGJYI3ZZ27H7EWT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"edf8b7debda7d871bede1282a9bae4f80973ea9a32d7be9ce743d87d063ed0ab","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-19T22:44:00Z","title_canon_sha256":"52afdca733a8dfa362cebcd2dac952722fa4bf4ceb7641e8fb2e5466b1a7cf55"},"schema_version":"1.0","source":{"id":"2405.12250","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.12250","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"arxiv_version","alias_value":"2405.12250v1","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.12250","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_12","alias_value":"2WHKOAPRDIVF","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_16","alias_value":"2WHKOAPRDIVFGJYI","created_at":"2026-07-05T08:21:17Z"},{"alias_kind":"pith_short_8","alias_value":"2WHKOAPR","created_at":"2026-07-05T08:21:17Z"}],"graph_snapshots":[{"event_id":"sha256:f00d116536e8e9132f48329e203b76c8135a20a5a88e5424f1c5827c2b54b589","target":"graph","created_at":"2026-07-05T08:21:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.12250/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper reveals a novel linear characteristic exclusive to transformer decoders, including models such as GPT, LLaMA, OPT, BLOOM and others. We analyze embedding transformations between sequential layers, uncovering a near-perfect linear relationship (Procrustes similarity score of 0.99). However, linearity decreases when the residual component is removed due to a consistently low output norm of the transformer layer. Our experiments show that removing or linearly approximating some of the most linear blocks of transformers does not affect significantly the loss or model performance. Moreov","authors_text":"Andrey Kuznetsov, Anton Razzhigaev, Denis Dimitrov, Elizaveta Goncharova, Ivan Oseledets, Matvey Mikhalchuk, Nikolai Gerasimenko","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-19T22:44:00Z","title":"Your Transformer is Secretly Linear"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.12250","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:06a8617dd51baa712e5ec298df3ee8a83f1bc96ad7b2591109aae9bf3b7b0a42","target":"record","created_at":"2026-07-05T08:21:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"edf8b7debda7d871bede1282a9bae4f80973ea9a32d7be9ce743d87d063ed0ab","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-19T22:44:00Z","title_canon_sha256":"52afdca733a8dfa362cebcd2dac952722fa4bf4ceb7641e8fb2e5466b1a7cf55"},"schema_version":"1.0","source":{"id":"2405.12250","kind":"arxiv","version":1}},"canonical_sha256":"d58ea701f11a2a532708de73af9fe4b4e45fac258a4bc422ae5e7ce6b77bc49c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d58ea701f11a2a532708de73af9fe4b4e45fac258a4bc422ae5e7ce6b77bc49c","first_computed_at":"2026-07-05T08:21:17.504867Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:21:17.504867Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"pq8YgQZCeV+R7kr8lVhiIbKWVYb0OxFsTE85m0LSSdricRCJlyzX8ume6x0XuI21419fUk+wpxA7vwiDEXVmCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:21:17.505298Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.12250","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:06a8617dd51baa712e5ec298df3ee8a83f1bc96ad7b2591109aae9bf3b7b0a42","sha256:f00d116536e8e9132f48329e203b76c8135a20a5a88e5424f1c5827c2b54b589"],"state_sha256":"9e19b5fbdbd8911ea923cea200455416c868cc1ec3b4af2ae4bab4b2e73dc1d6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6eywt7vq4NA6sVpXmsDKkcIdfmLnnWyLPAP46mxVfPjKqYBGXrysBI1IBLvpI37wlAHVZFhbrgxZaC/CYgMrDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-16T23:58:15.986038Z","bundle_sha256":"a16a174418af61e5a74611f5877a881c1dd85a7ef3c16ee8ec68558ac6202571"}}