{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:STMWE4ZCNCWMGDMSKQQ6UQJDFR","short_pith_number":"pith:STMWE4ZC","schema_version":"1.0","canonical_sha256":"94d962732268acc30d925421ea41232c445a46abd3ac72748adb88c0a551714e","source":{"kind":"arxiv","id":"2210.12023","version":3},"attestation_state":"computed","paper":{"title":"A Causal Framework to Quantify the Robustness of Mathematical Reasoning with Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alessandro Stolfo, Bernhard Sch\\\"olkopf, Kumar Shridhar, Mrinmaya Sachan, Zhijing Jin","submitted_at":"2022-10-21T15:12:37Z","abstract_excerpt":"We have recently witnessed a number of impressive results on hard mathematical reasoning problems with language models. At the same time, the robustness of these models has also been called into question; recent works have shown that models can rely on shallow patterns in the problem description when generating a solution. Building on the idea of behavioral testing, we propose a novel framework, which pins down the causal effect of various factors in the input, e.g., the surface form of the problem text, the operands, and math operators on the output solution. By grounding the behavioral analy"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2210.12023","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-10-21T15:12:37Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"c652a25733a48ac9cdee349959ba2697e64ae8ec191c8b051137f6a93db261d7","abstract_canon_sha256":"f93f69f839d487ba33c65aaf147052f1668a258034925a7944fa411b95850cee"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:18:36.690777Z","signature_b64":"5xsIyO2WOV5zixlrWZmE7oyWnkl7ucNVxKRU2oW/LgpSDqoy3JcAzX+LE4oRa156T0nhtlvKEF7CVlDUPRRbAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"94d962732268acc30d925421ea41232c445a46abd3ac72748adb88c0a551714e","last_reissued_at":"2026-07-05T06:18:36.690275Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:18:36.690275Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Causal Framework to Quantify the Robustness of Mathematical Reasoning with Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alessandro Stolfo, Bernhard Sch\\\"olkopf, Kumar Shridhar, Mrinmaya Sachan, Zhijing Jin","submitted_at":"2022-10-21T15:12:37Z","abstract_excerpt":"We have recently witnessed a number of impressive results on hard mathematical reasoning problems with language models. At the same time, the robustness of these models has also been called into question; recent works have shown that models can rely on shallow patterns in the problem description when generating a solution. Building on the idea of behavioral testing, we propose a novel framework, which pins down the causal effect of various factors in the input, e.g., the surface form of the problem text, the operands, and math operators on the output solution. By grounding the behavioral analy"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.12023","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.12023/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2210.12023","created_at":"2026-07-05T06:18:36.690345+00:00"},{"alias_kind":"arxiv_version","alias_value":"2210.12023v3","created_at":"2026-07-05T06:18:36.690345+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.12023","created_at":"2026-07-05T06:18:36.690345+00:00"},{"alias_kind":"pith_short_12","alias_value":"STMWE4ZCNCWM","created_at":"2026-07-05T06:18:36.690345+00:00"},{"alias_kind":"pith_short_16","alias_value":"STMWE4ZCNCWMGDMS","created_at":"2026-07-05T06:18:36.690345+00:00"},{"alias_kind":"pith_short_8","alias_value":"STMWE4ZC","created_at":"2026-07-05T06:18:36.690345+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.09881","citing_title":"Toward Calibrated, Fair, and accurate Deepfake Detection","ref_index":25,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR","json":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR.json","graph_json":"https://pith.science/api/pith-number/STMWE4ZCNCWMGDMSKQQ6UQJDFR/graph.json","events_json":"https://pith.science/api/pith-number/STMWE4ZCNCWMGDMSKQQ6UQJDFR/events.json","paper":"https://pith.science/paper/STMWE4ZC"},"agent_actions":{"view_html":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR","download_json":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR.json","view_paper":"https://pith.science/paper/STMWE4ZC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2210.12023&json=true","fetch_graph":"https://pith.science/api/pith-number/STMWE4ZCNCWMGDMSKQQ6UQJDFR/graph.json","fetch_events":"https://pith.science/api/pith-number/STMWE4ZCNCWMGDMSKQQ6UQJDFR/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR/action/timestamp_anchor","attest_storage":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR/action/storage_attestation","attest_author":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR/action/author_attestation","sign_citation":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR/action/citation_signature","submit_replication":"https://pith.science/pith/STMWE4ZCNCWMGDMSKQQ6UQJDFR/action/replication_record"}},"created_at":"2026-07-05T06:18:36.690345+00:00","updated_at":"2026-07-05T06:18:36.690345+00:00"}