{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BOZBDNTQTSVGQ3OV3K34FZNYHQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"82870184c474010992b21114f4f6a26d9e67b812d579908e14bf4a1353f907c0","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T14:39:33Z","title_canon_sha256":"e103455cf7c83326169aaa95e18f53b32cf2ecf649d774e9bcbbffd9d9379194"},"schema_version":"1.0","source":{"id":"2410.07985","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.07985","created_at":"2026-05-17T23:38:52Z"},{"alias_kind":"arxiv_version","alias_value":"2410.07985v3","created_at":"2026-05-17T23:38:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07985","created_at":"2026-05-17T23:38:52Z"},{"alias_kind":"pith_short_12","alias_value":"BOZBDNTQTSVG","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"BOZBDNTQTSVGQ3OV","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"BOZBDNTQ","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:6f69d37d806d93bfa4935aacfae3e3f7f390c54fd9a2be688e169d808beeb1f7","target":"graph","created_at":"2026-05-17T23:38:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"even the most advanced models, OpenAI o1-mini and OpenAI o1-preview, struggle with highly challenging Olympiad-level problems, with 60.54% and 52.55% accuracy"},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The 4428 problems constitute a fair, unbiased, and comprehensive sample of Olympiad-level mathematics, with human annotation free of selection bias or verification errors."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Omni-MATH supplies 4428 human-verified Olympiad math problems that expose top LLMs achieving only 52.55% to 60.54% accuracy on the most difficult items."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"A new benchmark of 4428 Olympiad math problems shows even top models like o1-preview reach only 52.55% accuracy."}],"snapshot_sha256":"295e7fa6fb1bef9d8f966868a98ad7c8d9c16be9a13b32cbfc20acd9ef01dfe0"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Recent advancements in large language models (LLMs) have led to significant breakthroughs in mathematical reasoning capabilities. However, existing benchmarks like GSM8K or MATH are now being solved with high accuracy (e.g., OpenAI o1 achieves 94.8\\% on MATH dataset), indicating their inadequacy for truly challenging these models. To bridge this gap, we propose a comprehensive and challenging benchmark specifically designed to assess LLMs' mathematical reasoning at the Olympiad level. Unlike existing Olympiad-related benchmarks, our dataset focuses exclusively on mathematics and comprises a va","authors_text":"Baobao Chang, Benyou Wang, Bofei Gao, Chenghao Ma, Daoguang Zan, Feifan Song, Ge Zhang, Lei Li, Lei Sha, Liang Chen, Qingxiu Dong, Runxin Xu, Shanghaoran Quan, Tianyu Liu, Xuancheng Ren, Yibo Miao, Yichang Zhang, Zefan Cai, Zhengyang Tang, Zhe Yang","cross_cats":[],"headline":"A new benchmark of 4428 Olympiad math problems shows even top models like o1-preview reach only 52.55% accuracy.","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T14:39:33Z","title":"Omni-MATH: A Universal Olympiad Level Mathematic Benchmark For Large Language Models"},"references":{"count":77,"internal_anchors":15,"resolved_work":77,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Training Verifiers to Solve Math Word Problems , author=. 2021 , eprint=","work_id":"0d46a05b-859d-423c-ae95-e7bb4f120561","year":2021},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Measuring Mathematical Problem Solving With the MATH Dataset , author=. 2021 , eprint=","work_id":"96984740-1b0a-4e03-9a02-9a0f6b7c8314","year":2021},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Have LLMs Advanced Enough? A Challenging Problem Solving Benchmark For Large Language Models , author=. 2023 , eprint=","work_id":"b4fa534b-c783-45a0-b643-85cee2c4d782","year":2023},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"MiniF2F: a cross-system benchmark for formal Olympiad-level mathematics , author=. 2022 , eprint=","work_id":"70cbb291-0c52-4b71-9a74-14949ec2d28b","year":2022},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"ProofNet: Autoformalizing and Formally Proving Undergraduate-Level Mathematics , author=. 2023 , eprint=","work_id":"186db717-bd16-4a71-9357-e93fdf5c2941","year":2023}],"snapshot_sha256":"ea96e6fad5a9729ddf9a4593fb01df5b0cc3d64fd83aa8084bfb087f22ed3f72"},"source":{"id":"2410.07985","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-15T09:04:10.133306Z","id":"0e53d206-3ba1-4673-84e9-33b811ac3427","model_set":{"reader":"grok-4.3"},"one_line_summary":"Omni-MATH supplies 4428 human-verified Olympiad math problems that expose top LLMs achieving only 52.55% to 60.54% accuracy on the most difficult items.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"A new benchmark of 4428 Olympiad math problems shows even top models like o1-preview reach only 52.55% accuracy.","strongest_claim":"even the most advanced models, OpenAI o1-mini and OpenAI o1-preview, struggle with highly challenging Olympiad-level problems, with 60.54% and 52.55% accuracy","weakest_assumption":"The 4428 problems constitute a fair, unbiased, and comprehensive sample of Olympiad-level mathematics, with human annotation free of selection bias or verification errors."}},"verdict_id":"0e53d206-3ba1-4673-84e9-33b811ac3427"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53028fcd875e12ec355d726e072c7a04f58461b312f1f072587f5a0590b543a4","target":"record","created_at":"2026-05-17T23:38:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"82870184c474010992b21114f4f6a26d9e67b812d579908e14bf4a1353f907c0","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T14:39:33Z","title_canon_sha256":"e103455cf7c83326169aaa95e18f53b32cf2ecf649d774e9bcbbffd9d9379194"},"schema_version":"1.0","source":{"id":"2410.07985","kind":"arxiv","version":3}},"canonical_sha256":"0bb211b6709caa686dd5dab7c2e5b83c3018ee224315ed0473d3973dd3e1623b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0bb211b6709caa686dd5dab7c2e5b83c3018ee224315ed0473d3973dd3e1623b","first_computed_at":"2026-05-17T23:38:52.937178Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:52.937178Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yAJ9gIHyNMj4pD4ELp4bktTHpVI8SFnLFTt9mTI007oU6uIU/kUwYitAi8y33nIAi0gT2i8KpxBAyGeEkh+qDg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:52.937760Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.07985","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53028fcd875e12ec355d726e072c7a04f58461b312f1f072587f5a0590b543a4","sha256:6f69d37d806d93bfa4935aacfae3e3f7f390c54fd9a2be688e169d808beeb1f7"],"state_sha256":"9e30b373728fb5878040283a03018c0c805c2394b05a9528cc5d7882e06858b3"}