{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:XJS4KDVO5PSVP3L76RXU7RDLGE","short_pith_number":"pith:XJS4KDVO","schema_version":"1.0","canonical_sha256":"ba65c50eaeebe557ed7ff46f4fc46b31112645d3e7be804f6f64309742d4a55d","source":{"kind":"arxiv","id":"2212.10559","version":3},"attestation_state":"computed","paper":{"title":"Why Can GPT Learn In-Context? Language Models Implicitly Perform Gradient Descent as Meta-Optimizers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Damai Dai, Furu Wei, Li Dong, Shuming Ma, Yaru Hao, Yutao Sun, Zhifang Sui","submitted_at":"2022-12-20T18:58:48Z","abstract_excerpt":"Large pretrained language models have shown surprising in-context learning (ICL) ability. With a few demonstration input-label pairs, they can predict the label for an unseen input without parameter updates. Despite the great success in performance, its working mechanism still remains an open question. In this paper, we explain language models as meta-optimizers and understand in-context learning as implicit finetuning. Theoretically, we figure out that Transformer attention has a dual form of gradient descent. On top of it, we understand ICL as follows: GPT first produces meta-gradients accor"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2212.10559","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-20T18:58:48Z","cross_cats_sorted":[],"title_canon_sha256":"85d5aae217fc86bf6ff9e7bd1e1a30ae1be7a7d461df81fcf885830126b163a9","abstract_canon_sha256":"5572a489d43f509b6ed9fc86fb460edff1b60e1cb01b66aebe0669af7d091d60"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:09:47.320219Z","signature_b64":"F1QXFAa574HBFCWlS61v7td0ZVwe5/uiQ8D+S0HpEQg+fs8MnVg0L6JGZ2+AWZq4LB0s13LhTBEwbskzRgz4BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ba65c50eaeebe557ed7ff46f4fc46b31112645d3e7be804f6f64309742d4a55d","last_reissued_at":"2026-07-05T06:09:47.319664Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:09:47.319664Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Why Can GPT Learn In-Context? Language Models Implicitly Perform Gradient Descent as Meta-Optimizers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Damai Dai, Furu Wei, Li Dong, Shuming Ma, Yaru Hao, Yutao Sun, Zhifang Sui","submitted_at":"2022-12-20T18:58:48Z","abstract_excerpt":"Large pretrained language models have shown surprising in-context learning (ICL) ability. With a few demonstration input-label pairs, they can predict the label for an unseen input without parameter updates. Despite the great success in performance, its working mechanism still remains an open question. In this paper, we explain language models as meta-optimizers and understand in-context learning as implicit finetuning. Theoretically, we figure out that Transformer attention has a dual form of gradient descent. On top of it, we understand ICL as follows: GPT first produces meta-gradients accor"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.10559","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.10559/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2212.10559","created_at":"2026-07-05T06:09:47.319736+00:00"},{"alias_kind":"arxiv_version","alias_value":"2212.10559v3","created_at":"2026-07-05T06:09:47.319736+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.10559","created_at":"2026-07-05T06:09:47.319736+00:00"},{"alias_kind":"pith_short_12","alias_value":"XJS4KDVO5PSV","created_at":"2026-07-05T06:09:47.319736+00:00"},{"alias_kind":"pith_short_16","alias_value":"XJS4KDVO5PSVP3L7","created_at":"2026-07-05T06:09:47.319736+00:00"},{"alias_kind":"pith_short_8","alias_value":"XJS4KDVO","created_at":"2026-07-05T06:09:47.319736+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":14,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2607.01918","citing_title":"Zeus: Towards Tuning-Free Foundation Model for Time Series Analysis","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20495","citing_title":"A Human-in-the-Loop Framework for Efficient Prompt Selection in Microscopy Vision-Language Models","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2502.10248","citing_title":"Step-Video-T2V Technical Report: The Practice, Challenges, and Future of Video Foundation Model","ref_index":112,"is_internal_anchor":false},{"citing_arxiv_id":"2303.17491","citing_title":"Language Models can Solve Computer Tasks","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2309.16797","citing_title":"Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution","ref_index":86,"is_internal_anchor":false},{"citing_arxiv_id":"2309.17421","citing_title":"The Dawn of LMMs: Preliminary Explorations with GPT-4V(ision)","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2305.03726","citing_title":"Otter: A Multi-Modal Model with In-Context Instruction Tuning","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12412","citing_title":"Stories in Space: In-Context Learning Trajectories in Conceptual Belief Space","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09727","citing_title":"One for All: A Non-Linear Transformer can Enable Cross-Domain Generalization for In-Context Reinforcement Learning","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10640","citing_title":"Towards Understanding Continual Factual Knowledge Acquisition of Language Models: From Theory to Algorithm","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23371","citing_title":"When Context Sticks: Studying Interference in In-Context Learning","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08537","citing_title":"Meta-learning In-Context Enables Training-Free Cross Subject Brain Decoding","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2303.18223","citing_title":"A Survey of Large Language Models","ref_index":67,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13403","citing_title":"Why Multimodal In-Context Learning Lags Behind? Unveiling the Inner Mechanisms and Bottlenecks","ref_index":9,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE","json":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE.json","graph_json":"https://pith.science/api/pith-number/XJS4KDVO5PSVP3L76RXU7RDLGE/graph.json","events_json":"https://pith.science/api/pith-number/XJS4KDVO5PSVP3L76RXU7RDLGE/events.json","paper":"https://pith.science/paper/XJS4KDVO"},"agent_actions":{"view_html":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE","download_json":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE.json","view_paper":"https://pith.science/paper/XJS4KDVO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2212.10559&json=true","fetch_graph":"https://pith.science/api/pith-number/XJS4KDVO5PSVP3L76RXU7RDLGE/graph.json","fetch_events":"https://pith.science/api/pith-number/XJS4KDVO5PSVP3L76RXU7RDLGE/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE/action/storage_attestation","attest_author":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE/action/author_attestation","sign_citation":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE/action/citation_signature","submit_replication":"https://pith.science/pith/XJS4KDVO5PSVP3L76RXU7RDLGE/action/replication_record"}},"created_at":"2026-07-05T06:09:47.319736+00:00","updated_at":"2026-07-05T06:09:47.319736+00:00"}