{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:ABW3DSUCNZ7M52IGJ25UQBLVX6","short_pith_number":"pith:ABW3DSUC","schema_version":"1.0","canonical_sha256":"006db1ca826e7ecee9064ebb480575bf8c096c98c71b2ced0446141c6c079752","source":{"kind":"arxiv","id":"2409.12866","version":2},"attestation_state":"computed","paper":{"title":"SpecEval: Evaluating Code Comprehension in Large Language Models via Program Specifications","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Lei Bu, Lezhi Ma, Shangqing Liu, Shangru Li, Yang Liu, Yida Wang","submitted_at":"2024-09-19T16:08:39Z","abstract_excerpt":"Large Language models have achieved impressive performance in automated software engineering. Extensive efforts have been made to evaluate the abilities of code LLMs in various aspects, with an increasing number of benchmarks and evaluation frameworks proposed. Apart from the most sought-after capability of code generation, the capability of code comprehension is being granted growing attention. Nevertheless, existing works assessing the code comprehension capability of LLMs exhibit varied limitations. Evaluation frameworks like CRUXEval and REval usually focus on code reasoning tasks over a c"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2409.12866","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-09-19T16:08:39Z","cross_cats_sorted":[],"title_canon_sha256":"d759064671fc786dfe09252139ca7f771522ceaf6f3025f8628a619a367457e6","abstract_canon_sha256":"38394ab520e1db2d373e672cfe5424809b25c2b34bba75e8cf9890c0c886a364"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:37:33.272953Z","signature_b64":"GRkvtjvS3tm8z0ltVOcPvfOtFWUi8cUjNShi08xcUdO5NZxbwj3/ExqHcVV5w28K1Xzi+29PuP8lzdqCXivbBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"006db1ca826e7ecee9064ebb480575bf8c096c98c71b2ced0446141c6c079752","last_reissued_at":"2026-07-05T10:37:33.271873Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:37:33.271873Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"SpecEval: Evaluating Code Comprehension in Large Language Models via Program Specifications","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Lei Bu, Lezhi Ma, Shangqing Liu, Shangru Li, Yang Liu, Yida Wang","submitted_at":"2024-09-19T16:08:39Z","abstract_excerpt":"Large Language models have achieved impressive performance in automated software engineering. Extensive efforts have been made to evaluate the abilities of code LLMs in various aspects, with an increasing number of benchmarks and evaluation frameworks proposed. Apart from the most sought-after capability of code generation, the capability of code comprehension is being granted growing attention. Nevertheless, existing works assessing the code comprehension capability of LLMs exhibit varied limitations. Evaluation frameworks like CRUXEval and REval usually focus on code reasoning tasks over a c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.12866","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.12866/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2409.12866","created_at":"2026-07-05T10:37:33.272019+00:00"},{"alias_kind":"arxiv_version","alias_value":"2409.12866v2","created_at":"2026-07-05T10:37:33.272019+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.12866","created_at":"2026-07-05T10:37:33.272019+00:00"},{"alias_kind":"pith_short_12","alias_value":"ABW3DSUCNZ7M","created_at":"2026-07-05T10:37:33.272019+00:00"},{"alias_kind":"pith_short_16","alias_value":"ABW3DSUCNZ7M52IG","created_at":"2026-07-05T10:37:33.272019+00:00"},{"alias_kind":"pith_short_8","alias_value":"ABW3DSUC","created_at":"2026-07-05T10:37:33.272019+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.21339","citing_title":"KBSpec: LLM-driven Formal Specification Generation with Evolving Domain Knowledge Base","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00677","citing_title":"Evaluating the Architectural Reasoning Capabilities of LLM Provers via the Obfuscated Natural Number Game","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.21570","citing_title":"SpecSyn: LLM-based Synthesis and Refinement of Formal Specifications for Real-world Program Verification","ref_index":52,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6","json":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6.json","graph_json":"https://pith.science/api/pith-number/ABW3DSUCNZ7M52IGJ25UQBLVX6/graph.json","events_json":"https://pith.science/api/pith-number/ABW3DSUCNZ7M52IGJ25UQBLVX6/events.json","paper":"https://pith.science/paper/ABW3DSUC"},"agent_actions":{"view_html":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6","download_json":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6.json","view_paper":"https://pith.science/paper/ABW3DSUC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2409.12866&json=true","fetch_graph":"https://pith.science/api/pith-number/ABW3DSUCNZ7M52IGJ25UQBLVX6/graph.json","fetch_events":"https://pith.science/api/pith-number/ABW3DSUCNZ7M52IGJ25UQBLVX6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6/action/storage_attestation","attest_author":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6/action/author_attestation","sign_citation":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6/action/citation_signature","submit_replication":"https://pith.science/pith/ABW3DSUCNZ7M52IGJ25UQBLVX6/action/replication_record"}},"created_at":"2026-07-05T10:37:33.272019+00:00","updated_at":"2026-07-05T10:37:33.272019+00:00"}