{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6UOJ7OL354FGOSOOL4LDV33ISE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"420495cf40720bc9749884f2255eba0dd09b0aeb67905e1df339ad3415fd59cb","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2024-10-02T20:04:02Z","title_canon_sha256":"57592ccf9f1852bacd188b5d95a8351042d32cf463b3f5f6b444612516e33cda"},"schema_version":"1.0","source":{"id":"2410.01999","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.01999","created_at":"2026-07-05T10:46:38Z"},{"alias_kind":"arxiv_version","alias_value":"2410.01999v4","created_at":"2026-07-05T10:46:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.01999","created_at":"2026-07-05T10:46:38Z"},{"alias_kind":"pith_short_12","alias_value":"6UOJ7OL354FG","created_at":"2026-07-05T10:46:38Z"},{"alias_kind":"pith_short_16","alias_value":"6UOJ7OL354FGOSOO","created_at":"2026-07-05T10:46:38Z"},{"alias_kind":"pith_short_8","alias_value":"6UOJ7OL3","created_at":"2026-07-05T10:46:38Z"}],"graph_snapshots":[{"event_id":"sha256:b4c69b80c12826149051fb653b1f590692f9c0ebac666a18da42bba4805aca74","target":"graph","created_at":"2026-07-05T10:46:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.01999/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advances in Code Large Language Models (CodeLLMs) have primarily focused on open-ended code generation, often overlooking the crucial aspect of code understanding and reasoning. To bridge this gap, we introduce CodeMMLU, a comprehensive multiple-choice benchmark designed to evaluate the depth of software and code comprehension in LLMs. CodeMMLU includes nearly 20,000 questions spanning diverse domains, including code analysis, defect detection, and software engineering principles across multiple programming languages. Unlike traditional benchmarks that emphasize code generation, CodeMML","authors_text":"Dung Nguyen Manh, Nam Le Hai, Nam V. Nguyen, Nghi D. Q. Bui, Quang Pham, Thang Phan Chau, Thong T. Doan","cross_cats":[],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2024-10-02T20:04:02Z","title":"CodeMMLU: A Multi-Task Benchmark for Assessing Code Understanding & Reasoning Capabilities of CodeLLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.01999","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:942af8a9fadbcb6117eaa2fa2ecda8efd419678aee69d1c9924a107e991bb5ab","target":"record","created_at":"2026-07-05T10:46:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"420495cf40720bc9749884f2255eba0dd09b0aeb67905e1df339ad3415fd59cb","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2024-10-02T20:04:02Z","title_canon_sha256":"57592ccf9f1852bacd188b5d95a8351042d32cf463b3f5f6b444612516e33cda"},"schema_version":"1.0","source":{"id":"2410.01999","kind":"arxiv","version":4}},"canonical_sha256":"f51c9fb97bef0a6749ce5f163aef689125ce3d2eb550b88e850d79fbe9856188","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f51c9fb97bef0a6749ce5f163aef689125ce3d2eb550b88e850d79fbe9856188","first_computed_at":"2026-07-05T10:46:38.656378Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:46:38.656378Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mwSOXfZkwiX+LjLLeerHprF7WiMITc0bdiCRyGNZz5MDGcIqXIlyCqTU4O27vDAMaG7TsZQD024mDTZaCiobDw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:46:38.656838Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.01999","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:942af8a9fadbcb6117eaa2fa2ecda8efd419678aee69d1c9924a107e991bb5ab","sha256:b4c69b80c12826149051fb653b1f590692f9c0ebac666a18da42bba4805aca74"],"state_sha256":"9b69ed5c60c7ac6759021ba026dd69f8bbf31eba94dfadfe0add1a9c664490aa"}