{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:BZKHMBUM2YXIK7FTZ4JLEP55SH","short_pith_number":"pith:BZKHMBUM","schema_version":"1.0","canonical_sha256":"0e5476068cd62e857cb3cf12b23fbd91cdc2548a7ada9bd683072e57dbce6bb5","source":{"kind":"arxiv","id":"2305.18654","version":3},"attestation_state":"computed","paper":{"title":"Faith and Fate: Limits of Transformers on Compositionality","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Allyson Ettinger, Bill Yuchen Lin, Chandra Bhagavatula, Jena D. Hwang, Liwei Jiang, Melanie Sclar, Nouha Dziri, Peter West, Ronan Le Bras, Sean Welleck, Soumya Sanyal, Xiang Lorraine Li, Xiang Ren, Ximing Lu, Yejin Choi, Zaid Harchaoui","submitted_at":"2023-05-29T23:24:14Z","abstract_excerpt":"Transformer large language models (LLMs) have sparked admiration for their exceptional performance on tasks that demand intricate multi-step reasoning. Yet, these models simultaneously show failures on surprisingly trivial problems. This begs the question: Are these errors incidental, or do they signal more substantial limitations? In an attempt to demystify transformer LLMs, we investigate the limits of these models across three representative compositional tasks -- multi-digit multiplication, logic grid puzzles, and a classic dynamic programming problem. These tasks require breaking problems"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.18654","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T23:24:14Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"215c65f58bc1a4f34da8b2d67ac4cbdc27df985efaf3d9106adcd270e669d890","abstract_canon_sha256":"7ae42a78985f5ae7b75ac9a2f35e4d7eb7e629c8124f4f14c04f27d22464e2b6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:07:11.029237Z","signature_b64":"6y2XBx7O4WPc23mYE6ApZicP6JQ7WMinH3wXKKpx+asIeiUTcrxLEcvMkBx+0pLMxJWgkVcIIjhGvxBROsy3CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0e5476068cd62e857cb3cf12b23fbd91cdc2548a7ada9bd683072e57dbce6bb5","last_reissued_at":"2026-07-05T07:07:11.028756Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:07:11.028756Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Faith and Fate: Limits of Transformers on Compositionality","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Allyson Ettinger, Bill Yuchen Lin, Chandra Bhagavatula, Jena D. Hwang, Liwei Jiang, Melanie Sclar, Nouha Dziri, Peter West, Ronan Le Bras, Sean Welleck, Soumya Sanyal, Xiang Lorraine Li, Xiang Ren, Ximing Lu, Yejin Choi, Zaid Harchaoui","submitted_at":"2023-05-29T23:24:14Z","abstract_excerpt":"Transformer large language models (LLMs) have sparked admiration for their exceptional performance on tasks that demand intricate multi-step reasoning. Yet, these models simultaneously show failures on surprisingly trivial problems. This begs the question: Are these errors incidental, or do they signal more substantial limitations? In an attempt to demystify transformer LLMs, we investigate the limits of these models across three representative compositional tasks -- multi-digit multiplication, logic grid puzzles, and a classic dynamic programming problem. These tasks require breaking problems"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.18654","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.18654/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.18654","created_at":"2026-07-05T07:07:11.028808+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.18654v3","created_at":"2026-07-05T07:07:11.028808+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.18654","created_at":"2026-07-05T07:07:11.028808+00:00"},{"alias_kind":"pith_short_12","alias_value":"BZKHMBUM2YXI","created_at":"2026-07-05T07:07:11.028808+00:00"},{"alias_kind":"pith_short_16","alias_value":"BZKHMBUM2YXIK7FT","created_at":"2026-07-05T07:07:11.028808+00:00"},{"alias_kind":"pith_short_8","alias_value":"BZKHMBUM","created_at":"2026-07-05T07:07:11.028808+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":11,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.08403","citing_title":"Game Theory Driven Multi-Agent Framework Mitigates Language Model Hallucination","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2606.21884","citing_title":"A Verifiable Search Is Not a Learnable Chain-of-Thought","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.17667","citing_title":"Handling Feature Heterogeneity with Learnable Graph Patches","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05106","citing_title":"Arithmetic Pedagogy for Language Models","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24756","citing_title":"Proper Scoring Rules for Agentic Uncertainty Quantification","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2505.24187","citing_title":"Beyond Exponential Decay: Rethinking Error Accumulation in Large Language Models","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2510.01685","citing_title":"How Do Language Models Compose Functions?","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2510.05307","citing_title":"When Should Users Check? Modeling Confirmation Frequency inMulti-Step Agentic AI Tasks","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2511.01101","citing_title":"TSVer: A Benchmark for Fact Verification Against Time-Series Evidence","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2604.25166","citing_title":"Training Transformers as a Universal Computer","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2403.07974","citing_title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","ref_index":95,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH","json":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH.json","graph_json":"https://pith.science/api/pith-number/BZKHMBUM2YXIK7FTZ4JLEP55SH/graph.json","events_json":"https://pith.science/api/pith-number/BZKHMBUM2YXIK7FTZ4JLEP55SH/events.json","paper":"https://pith.science/paper/BZKHMBUM"},"agent_actions":{"view_html":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH","download_json":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH.json","view_paper":"https://pith.science/paper/BZKHMBUM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.18654&json=true","fetch_graph":"https://pith.science/api/pith-number/BZKHMBUM2YXIK7FTZ4JLEP55SH/graph.json","fetch_events":"https://pith.science/api/pith-number/BZKHMBUM2YXIK7FTZ4JLEP55SH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH/action/storage_attestation","attest_author":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH/action/author_attestation","sign_citation":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH/action/citation_signature","submit_replication":"https://pith.science/pith/BZKHMBUM2YXIK7FTZ4JLEP55SH/action/replication_record"}},"created_at":"2026-07-05T07:07:11.028808+00:00","updated_at":"2026-07-05T07:07:11.028808+00:00"}