{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:XQK2CAFLS2D7L5PXWPWJKU7OHA","short_pith_number":"pith:XQK2CAFL","schema_version":"1.0","canonical_sha256":"bc15a100ab9687f5f5f7b3ec9553ee380496c8f74b69bebec7e9bd5b36da0768","source":{"kind":"arxiv","id":"2502.16906","version":1},"attestation_state":"computed","paper":{"title":"AutoLogi: Automated Generation of Logic Puzzles for Evaluating Reasoning Abilities of Large Language Models","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Bowen Yu, Fei Huang, Junyang Lin, Keming Lu, Qinyuan Cheng, Qin Zhu, Runyu Peng, Xipeng Qiu, Xuanjing Huang","submitted_at":"2025-02-24T07:02:31Z","abstract_excerpt":"While logical reasoning evaluation of Large Language Models (LLMs) has attracted significant attention, existing benchmarks predominantly rely on multiple-choice formats that are vulnerable to random guessing, leading to overestimated performance and substantial performance fluctuations. To obtain more accurate assessments of models' reasoning capabilities, we propose an automated method for synthesizing open-ended logic puzzles, and use it to develop a bilingual benchmark, AutoLogi. Our approach features program-based verification and controllable difficulty levels, enabling more reliable eva"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.16906","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-24T07:02:31Z","cross_cats_sorted":[],"title_canon_sha256":"3c3b3b6eff72e6b14c85cc80823ce14b198b631fb2434947ad36cd0a492fb976","abstract_canon_sha256":"420d234b33b312106d45f043bd2c5a73a06f9d62b7f17def358311959edffbc1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:19:02.014006Z","signature_b64":"JWEneG4AQepEXRd+aazyfcM07mjEaxFVrhJY5oDE0Zrnn1U8xQQk0/kIOwSWXhyvlda4kv+lHjBDl/UXwD21Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bc15a100ab9687f5f5f7b3ec9553ee380496c8f74b69bebec7e9bd5b36da0768","last_reissued_at":"2026-07-05T10:19:02.013412Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:19:02.013412Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"AutoLogi: Automated Generation of Logic Puzzles for Evaluating Reasoning Abilities of Large Language Models","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Bowen Yu, Fei Huang, Junyang Lin, Keming Lu, Qinyuan Cheng, Qin Zhu, Runyu Peng, Xipeng Qiu, Xuanjing Huang","submitted_at":"2025-02-24T07:02:31Z","abstract_excerpt":"While logical reasoning evaluation of Large Language Models (LLMs) has attracted significant attention, existing benchmarks predominantly rely on multiple-choice formats that are vulnerable to random guessing, leading to overestimated performance and substantial performance fluctuations. To obtain more accurate assessments of models' reasoning capabilities, we propose an automated method for synthesizing open-ended logic puzzles, and use it to develop a bilingual benchmark, AutoLogi. Our approach features program-based verification and controllable difficulty levels, enabling more reliable eva"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.16906","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.16906/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.16906","created_at":"2026-07-05T10:19:02.013495+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.16906v1","created_at":"2026-07-05T10:19:02.013495+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.16906","created_at":"2026-07-05T10:19:02.013495+00:00"},{"alias_kind":"pith_short_12","alias_value":"XQK2CAFLS2D7","created_at":"2026-07-05T10:19:02.013495+00:00"},{"alias_kind":"pith_short_16","alias_value":"XQK2CAFLS2D7L5PX","created_at":"2026-07-05T10:19:02.013495+00:00"},{"alias_kind":"pith_short_8","alias_value":"XQK2CAFL","created_at":"2026-07-05T10:19:02.013495+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":12,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.26671","citing_title":"NebulaExp-8B: An Empirical Post-Training Pipeline via Full-Scale Ablation Research","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2606.15079","citing_title":"Ling and Ring 2.6 Technical Report: Efficient and Instant Agentic Intelligence at Trillion-Parameter Scale","ref_index":178,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08068","citing_title":"DICE: Entropy-Regularized Equilibrium Selection for Stable Multi-Agent LLM Coordination","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05464","citing_title":"Step-by-Step Optimization-like Reasoning in LLMs over Expanding Search Spaces","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05201","citing_title":"State commitment learning: training language models to distinguish computation from memory","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30626","citing_title":"DOPD: Dual On-policy Distillation","ref_index":62,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07731","citing_title":"Benchmarking EngGPT2-16B-A3B against Comparable Italian and International Open-source LLMs","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15207","citing_title":"TeamTR: Trust-Region Fine-Tuning for Multi-Agent LLM Coordination","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2602.02188","citing_title":"Reasoning in a Combinatorial and Constrained World: Benchmarking LLMs on Natural-Language Combinatorial Optimization","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07731","citing_title":"Benchmarking EngGPT2-16B-A3B against Comparable Italian and International Open-source LLMs","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05216","citing_title":"SAT: Sequential Agent Tuning for Coordinator Free Plug and Play Multi-LLM Training with Monotonic Improvement Guarantees","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2505.09388","citing_title":"Qwen3 Technical Report","ref_index":41,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA","json":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA.json","graph_json":"https://pith.science/api/pith-number/XQK2CAFLS2D7L5PXWPWJKU7OHA/graph.json","events_json":"https://pith.science/api/pith-number/XQK2CAFLS2D7L5PXWPWJKU7OHA/events.json","paper":"https://pith.science/paper/XQK2CAFL"},"agent_actions":{"view_html":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA","download_json":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA.json","view_paper":"https://pith.science/paper/XQK2CAFL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.16906&json=true","fetch_graph":"https://pith.science/api/pith-number/XQK2CAFLS2D7L5PXWPWJKU7OHA/graph.json","fetch_events":"https://pith.science/api/pith-number/XQK2CAFLS2D7L5PXWPWJKU7OHA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA/action/storage_attestation","attest_author":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA/action/author_attestation","sign_citation":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA/action/citation_signature","submit_replication":"https://pith.science/pith/XQK2CAFLS2D7L5PXWPWJKU7OHA/action/replication_record"}},"created_at":"2026-07-05T10:19:02.013495+00:00","updated_at":"2026-07-05T10:19:02.013495+00:00"}