{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:EFBFVPDILPDKE6JR4UC7B5XNKW","short_pith_number":"pith:EFBFVPDI","schema_version":"1.0","canonical_sha256":"21425abc685bc6a27931e505f0f6ed55a18274df20d6958b07a7701d93c6e750","source":{"kind":"arxiv","id":"2502.00855","version":1},"attestation_state":"computed","paper":{"title":"Psychometric-Based Evaluation for Theorem Proving with Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Feng Yang, Jianyu Zhang, Jilin Hu, Long Zhang, Xiaokun Luan, Yongwang Zhao, Zhiwei Xu","submitted_at":"2025-02-02T17:00:22Z","abstract_excerpt":"Large language models (LLMs) for formal theorem proving have become a prominent research focus. At present, the proving ability of these LLMs is mainly evaluated through proof pass rates on datasets such as miniF2F. However, this evaluation method overlooks the varying importance of theorems. As a result, it fails to highlight the real performance disparities between LLMs and leads to high evaluation costs. This study proposes a psychometric-based evaluation method for theorem proving with LLMs, comprising two main components: Dataset Annotation and Adaptive Evaluation. First, we propose a met"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.00855","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-02-02T17:00:22Z","cross_cats_sorted":[],"title_canon_sha256":"024b24b5afc45303c55c72cd8fe8f5b7d8e9444a5b97427294bcd9142c749af9","abstract_canon_sha256":"139cf4197f938094035a2807f238e3ca0eeca498cef5e97d2de4e0e7fee0b15e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:08:39.312768Z","signature_b64":"wJWyFMlqTW/L4xqXBBH+wbDhP26c784hilDCDOY8mDzbqyyo6fk/NlCgsn2GUVB6vJiNDw5zCzBqn2jTLRk1Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"21425abc685bc6a27931e505f0f6ed55a18274df20d6958b07a7701d93c6e750","last_reissued_at":"2026-07-05T10:08:39.312390Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:08:39.312390Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Psychometric-Based Evaluation for Theorem Proving with Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Feng Yang, Jianyu Zhang, Jilin Hu, Long Zhang, Xiaokun Luan, Yongwang Zhao, Zhiwei Xu","submitted_at":"2025-02-02T17:00:22Z","abstract_excerpt":"Large language models (LLMs) for formal theorem proving have become a prominent research focus. At present, the proving ability of these LLMs is mainly evaluated through proof pass rates on datasets such as miniF2F. However, this evaluation method overlooks the varying importance of theorems. As a result, it fails to highlight the real performance disparities between LLMs and leads to high evaluation costs. This study proposes a psychometric-based evaluation method for theorem proving with LLMs, comprising two main components: Dataset Annotation and Adaptive Evaluation. First, we propose a met"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.00855","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.00855/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.00855","created_at":"2026-07-05T10:08:39.312446+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.00855v1","created_at":"2026-07-05T10:08:39.312446+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.00855","created_at":"2026-07-05T10:08:39.312446+00:00"},{"alias_kind":"pith_short_12","alias_value":"EFBFVPDILPDK","created_at":"2026-07-05T10:08:39.312446+00:00"},{"alias_kind":"pith_short_16","alias_value":"EFBFVPDILPDKE6JR","created_at":"2026-07-05T10:08:39.312446+00:00"},{"alias_kind":"pith_short_8","alias_value":"EFBFVPDI","created_at":"2026-07-05T10:08:39.312446+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW","json":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW.json","graph_json":"https://pith.science/api/pith-number/EFBFVPDILPDKE6JR4UC7B5XNKW/graph.json","events_json":"https://pith.science/api/pith-number/EFBFVPDILPDKE6JR4UC7B5XNKW/events.json","paper":"https://pith.science/paper/EFBFVPDI"},"agent_actions":{"view_html":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW","download_json":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW.json","view_paper":"https://pith.science/paper/EFBFVPDI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.00855&json=true","fetch_graph":"https://pith.science/api/pith-number/EFBFVPDILPDKE6JR4UC7B5XNKW/graph.json","fetch_events":"https://pith.science/api/pith-number/EFBFVPDILPDKE6JR4UC7B5XNKW/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW/action/timestamp_anchor","attest_storage":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW/action/storage_attestation","attest_author":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW/action/author_attestation","sign_citation":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW/action/citation_signature","submit_replication":"https://pith.science/pith/EFBFVPDILPDKE6JR4UC7B5XNKW/action/replication_record"}},"created_at":"2026-07-05T10:08:39.312446+00:00","updated_at":"2026-07-05T10:08:39.312446+00:00"}