{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:7MRDW5W4XKWX5HGCD43AGDBXWA","short_pith_number":"pith:7MRDW5W4","schema_version":"1.0","canonical_sha256":"fb223b76dcbaad7e9cc21f36030c37b0360c3e5894f4f802ac8f44ef9cc02e80","source":{"kind":"arxiv","id":"2502.06453","version":2},"attestation_state":"computed","paper":{"title":"MATH-Perturb: Benchmarking LLMs' Math Reasoning Abilities against Hard Perturbations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Chi Jin, Chiyuan Zhang, Hui Yuan, Jiacheng Guo, Jiawei Ge, Kaixuan Huang, Mengdi Wang, Ming Yin, Runzhe Wang, Shange Tang, Tianle Cai, Wenzhe Li, Xiang Ji, Xinyun Chen, Yangsibo Huang, Yingqing Guo, Yue Wu, Zihao Li","submitted_at":"2025-02-10T13:31:46Z","abstract_excerpt":"Large language models have demonstrated impressive performance on challenging mathematical reasoning tasks, which has triggered the discussion of whether the performance is achieved by true reasoning capability or memorization. To investigate this question, prior work has constructed mathematical benchmarks when questions undergo simple perturbations -- modifications that still preserve the underlying reasoning patterns of the solutions. However, no work has explored hard perturbations, which fundamentally change the nature of the problem so that the original solution steps do not apply. To br"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.06453","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-02-10T13:31:46Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"4bff9c2cf11e1c428219132ad0e8012386021ef1d332a86ebc99ce975b207f00","abstract_canon_sha256":"675a0a59efe37f45694118de3daaf8fbc4903c665cb2ffc3f03df8ff3a6c32fe"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:13:27.799829Z","signature_b64":"muzN0P8QXdsjDraU2aoA7HcikkwHggEXZ5wwGHbQnlLV+1Ve6sr9lWUIzJjN7pMiA1uBHJJeB9pHzO6EfWRJBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fb223b76dcbaad7e9cc21f36030c37b0360c3e5894f4f802ac8f44ef9cc02e80","last_reissued_at":"2026-07-05T10:13:27.799337Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:13:27.799337Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MATH-Perturb: Benchmarking LLMs' Math Reasoning Abilities against Hard Perturbations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Chi Jin, Chiyuan Zhang, Hui Yuan, Jiacheng Guo, Jiawei Ge, Kaixuan Huang, Mengdi Wang, Ming Yin, Runzhe Wang, Shange Tang, Tianle Cai, Wenzhe Li, Xiang Ji, Xinyun Chen, Yangsibo Huang, Yingqing Guo, Yue Wu, Zihao Li","submitted_at":"2025-02-10T13:31:46Z","abstract_excerpt":"Large language models have demonstrated impressive performance on challenging mathematical reasoning tasks, which has triggered the discussion of whether the performance is achieved by true reasoning capability or memorization. To investigate this question, prior work has constructed mathematical benchmarks when questions undergo simple perturbations -- modifications that still preserve the underlying reasoning patterns of the solutions. However, no work has explored hard perturbations, which fundamentally change the nature of the problem so that the original solution steps do not apply. To br"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.06453","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.06453/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.06453","created_at":"2026-07-05T10:13:27.799397+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.06453v2","created_at":"2026-07-05T10:13:27.799397+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.06453","created_at":"2026-07-05T10:13:27.799397+00:00"},{"alias_kind":"pith_short_12","alias_value":"7MRDW5W4XKWX","created_at":"2026-07-05T10:13:27.799397+00:00"},{"alias_kind":"pith_short_16","alias_value":"7MRDW5W4XKWX5HGC","created_at":"2026-07-05T10:13:27.799397+00:00"},{"alias_kind":"pith_short_8","alias_value":"7MRDW5W4","created_at":"2026-07-05T10:13:27.799397+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":15,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.08317","citing_title":"Blind-Spots-Bench: Evaluating Blind Spots in Multimodal Models","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2606.26986","citing_title":"ReaORE: Reasoning-Guided Progressive Open Relation Extraction Empowered by Large Reasoning Models","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2606.20227","citing_title":"QMFOL: Benchmarking Large Language Model Reasoning via Quantifiable Monadic First-Order Logic Test Case Generation","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03606","citing_title":"Testing LLM Arithmetic Reasoning Generalization with Automatic Numeric-Remapping Attacks","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01462","citing_title":"An Enigma of Artificial Reason: Investigating the Production-Evaluation Gap in Large Reasoning Models","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06840","citing_title":"Characterize Then Distill: Mechanistic Reasoning in Large Output Spaces","ref_index":119,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08571","citing_title":"Robust Reasoning Benchmark","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2605.16286","citing_title":"Homoglyph-based Adversarial Perturbation of Introductory Computer Science Theory Problems","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23135","citing_title":"Characterizing Paraphrase-Induced Failures in Lean 4 Autoformalization","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2509.17677","citing_title":"EngiBench: A Benchmark for Evaluating Large Language Models on Engineering Problem Solving","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2601.19924","citing_title":"OPT-Engine: Benchmarking the Limits of LLMs in Optimization Modeling via Complexity Scaling","ref_index":37,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08571","citing_title":"Robust Reasoning Benchmark","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11636","citing_title":"Seir\\^enes: Adversarial Self-Play with Evolving Distractions for LLM Reasoning","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2503.09567","citing_title":"Towards Reasoning Era: A Survey of Long Chain-of-Thought for Reasoning Large Language Models","ref_index":293,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23135","citing_title":"Characterizing Paraphrase-Induced Failures in Lean 4 Autoformalization","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA","json":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA.json","graph_json":"https://pith.science/api/pith-number/7MRDW5W4XKWX5HGCD43AGDBXWA/graph.json","events_json":"https://pith.science/api/pith-number/7MRDW5W4XKWX5HGCD43AGDBXWA/events.json","paper":"https://pith.science/paper/7MRDW5W4"},"agent_actions":{"view_html":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA","download_json":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA.json","view_paper":"https://pith.science/paper/7MRDW5W4","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.06453&json=true","fetch_graph":"https://pith.science/api/pith-number/7MRDW5W4XKWX5HGCD43AGDBXWA/graph.json","fetch_events":"https://pith.science/api/pith-number/7MRDW5W4XKWX5HGCD43AGDBXWA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA/action/storage_attestation","attest_author":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA/action/author_attestation","sign_citation":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA/action/citation_signature","submit_replication":"https://pith.science/pith/7MRDW5W4XKWX5HGCD43AGDBXWA/action/replication_record"}},"created_at":"2026-07-05T10:13:27.799397+00:00","updated_at":"2026-07-05T10:13:27.799397+00:00"}