{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:G3YE7KHLXRQFCZGFASSSCPCDIB","short_pith_number":"pith:G3YE7KHL","schema_version":"1.0","canonical_sha256":"36f04fa8ebbc605164c504a5213c434078840118d764ccb37bd35c2b8c6e2f84","source":{"kind":"arxiv","id":"2312.02143","version":3},"attestation_state":"computed","paper":{"title":"Competition-Level Problems are Effective LLM Evaluators","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Chen Lin, Fangyu Lei, Nan Duan, Shuai Lu, Weizhu Chen, Xiao Liu, Yaobo Liang, Yelong Shen, Yeyun Gong, Yiming Huang, Zhenghao Lin","submitted_at":"2023-12-04T18:58:57Z","abstract_excerpt":"Large language models (LLMs) have demonstrated impressive reasoning capabilities, yet there is ongoing debate about these abilities and the potential data contamination problem recently. This paper aims to evaluate the reasoning capacities of LLMs, specifically in solving recent competition-level programming problems in Codeforces, which are expert-crafted and unique, requiring deep understanding and robust reasoning skills. We first provide a comprehensive evaluation of GPT-4's peiceived zero-shot performance on this task, considering various aspects such as problems' release time, difficulti"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2312.02143","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-12-04T18:58:57Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9e5d4e0c929f6c8126f209a3a9e69f3286bd2cde468867d682d586449255ce93","abstract_canon_sha256":"70aa2a9bc3623683e7c9ba1b8feaf02e4ea5add111090355209cf0dda852fe0e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:27:00.802419Z","signature_b64":"IiL+SUuDSs1MRR1Rnn0GCjtdWnsIQEbpe4Opty1/I1UV6Z8Wg9kmVm6CQgUq6p7s0YaQKi48pfa6Mez/rX5fDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"36f04fa8ebbc605164c504a5213c434078840118d764ccb37bd35c2b8c6e2f84","last_reissued_at":"2026-07-05T08:27:00.801899Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:27:00.801899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Competition-Level Problems are Effective LLM Evaluators","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Chen Lin, Fangyu Lei, Nan Duan, Shuai Lu, Weizhu Chen, Xiao Liu, Yaobo Liang, Yelong Shen, Yeyun Gong, Yiming Huang, Zhenghao Lin","submitted_at":"2023-12-04T18:58:57Z","abstract_excerpt":"Large language models (LLMs) have demonstrated impressive reasoning capabilities, yet there is ongoing debate about these abilities and the potential data contamination problem recently. This paper aims to evaluate the reasoning capacities of LLMs, specifically in solving recent competition-level programming problems in Codeforces, which are expert-crafted and unique, requiring deep understanding and robust reasoning skills. We first provide a comprehensive evaluation of GPT-4's peiceived zero-shot performance on this task, considering various aspects such as problems' release time, difficulti"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.02143","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.02143/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2312.02143","created_at":"2026-07-05T08:27:00.801974+00:00"},{"alias_kind":"arxiv_version","alias_value":"2312.02143v3","created_at":"2026-07-05T08:27:00.801974+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.02143","created_at":"2026-07-05T08:27:00.801974+00:00"},{"alias_kind":"pith_short_12","alias_value":"G3YE7KHLXRQF","created_at":"2026-07-05T08:27:00.801974+00:00"},{"alias_kind":"pith_short_16","alias_value":"G3YE7KHLXRQFCZGF","created_at":"2026-07-05T08:27:00.801974+00:00"},{"alias_kind":"pith_short_8","alias_value":"G3YE7KHL","created_at":"2026-07-05T08:27:00.801974+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2406.04244","citing_title":"Benchmark Data Contamination of Large Language Models: A Survey","ref_index":65,"is_internal_anchor":false},{"citing_arxiv_id":"2512.21132","citing_title":"AutoBaxBuilder: Bootstrapping Code Security Benchmarking","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2507.22359","citing_title":"League of LLMs: A Benchmark-Free Paradigm for Mutual Evaluation of Large Language Models","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2509.17677","citing_title":"EngiBench: A Benchmark for Evaluating Large Language Models on Engineering Problem Solving","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2403.07974","citing_title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","ref_index":259,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB","json":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB.json","graph_json":"https://pith.science/api/pith-number/G3YE7KHLXRQFCZGFASSSCPCDIB/graph.json","events_json":"https://pith.science/api/pith-number/G3YE7KHLXRQFCZGFASSSCPCDIB/events.json","paper":"https://pith.science/paper/G3YE7KHL"},"agent_actions":{"view_html":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB","download_json":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB.json","view_paper":"https://pith.science/paper/G3YE7KHL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2312.02143&json=true","fetch_graph":"https://pith.science/api/pith-number/G3YE7KHLXRQFCZGFASSSCPCDIB/graph.json","fetch_events":"https://pith.science/api/pith-number/G3YE7KHLXRQFCZGFASSSCPCDIB/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB/action/timestamp_anchor","attest_storage":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB/action/storage_attestation","attest_author":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB/action/author_attestation","sign_citation":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB/action/citation_signature","submit_replication":"https://pith.science/pith/G3YE7KHLXRQFCZGFASSSCPCDIB/action/replication_record"}},"created_at":"2026-07-05T08:27:00.801974+00:00","updated_at":"2026-07-05T08:27:00.801974+00:00"}