{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:X7LZYKZRAUM6DO4YHMR6GAZAFD","short_pith_number":"pith:X7LZYKZR","canonical_record":{"source":{"id":"2309.01940","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T04:12:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"68b723c5f2a3e099fef2cac0a7ba515a929448b49cd56c5b4adbae8c6dc40497","abstract_canon_sha256":"baf224a4f8b2127e2a4db580917a6c69cdff7e07be142e83ed269bc84922453e"},"schema_version":"1.0"},"canonical_sha256":"bfd79c2b310519e1bb983b23e3032028ee4052e44926f557fa5c768ee10fc320","source":{"kind":"arxiv","id":"2309.01940","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.01940","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"arxiv_version","alias_value":"2309.01940v4","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.01940","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_12","alias_value":"X7LZYKZRAUM6","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_16","alias_value":"X7LZYKZRAUM6DO4Y","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_8","alias_value":"X7LZYKZR","created_at":"2026-07-05T07:54:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:X7LZYKZRAUM6DO4YHMR6GAZAFD","target":"record","payload":{"canonical_record":{"source":{"id":"2309.01940","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T04:12:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"68b723c5f2a3e099fef2cac0a7ba515a929448b49cd56c5b4adbae8c6dc40497","abstract_canon_sha256":"baf224a4f8b2127e2a4db580917a6c69cdff7e07be142e83ed269bc84922453e"},"schema_version":"1.0"},"canonical_sha256":"bfd79c2b310519e1bb983b23e3032028ee4052e44926f557fa5c768ee10fc320","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:54:32.627794Z","signature_b64":"csNfxx0c/aekVhP60IZR4A7ibhUGO26yhhvacsiWgbvINNiuxXCo98A0jYU+3bSn3KHbg6niGNReRLwKNUFGAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bfd79c2b310519e1bb983b23e3032028ee4052e44926f557fa5c768ee10fc320","last_reissued_at":"2026-07-05T07:54:32.627203Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:54:32.627203Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.01940","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:54:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QJ7+7QjY4Oc1JN/peuTJeIEB+jjReCQQbkaZw+65yTmkFCumuKMwpjT0+6uTUVP8IemY025lYpw/n2ur9xDnBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:16:35.939305Z"},"content_sha256":"90421286ea7956008eff326027425b9bfe13bbbcbf1f2bbad91f8536f44dffbd","schema_version":"1.0","event_id":"sha256:90421286ea7956008eff326027425b9bfe13bbbcbf1f2bbad91f8536f44dffbd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:X7LZYKZRAUM6DO4YHMR6GAZAFD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CodeApex: A Bilingual Programming Evaluation Benchmark for Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Huacan Chai, Jianghao Lin, Jiayi Lei, Jingkuan Wang, Kangning Zhang, Kounianhua Du, Lingyue Fu, Longteng Fan, Renting Rui, Shuang Luo, Siyuan Qi, Weiming Zhang, Weinan Zhang, Yifan Liu, Yong Yu, Yuchen Fang","submitted_at":"2023-09-05T04:12:01Z","abstract_excerpt":"With the emergence of Large Language Models (LLMs), there has been a significant improvement in the programming capabilities of models, attracting growing attention from researchers. Evaluating the programming capabilities of LLMs is crucial as it reflects the multifaceted abilities of LLMs, and it has numerous downstream applications. In this paper, we propose CodeApex, a bilingual benchmark dataset focusing on the programming comprehension, code generation, and code correction abilities of LLMs. Programming comprehension task tests LLMs on multiple-choice exam questions covering conceptual u"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.01940","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.01940/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:54:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FpkDfHgp/3GEOBGugpC1aMBgRq9L/rT3om6CE18gqY+joLTXSRKy36VGOwBSnwMyFe8CEVWTsLQ45RBvEGn0Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:16:35.939690Z"},"content_sha256":"e0dd0176e78461f6ae1db6226740b5abc1306f977695b116e137edb173ef5c72","schema_version":"1.0","event_id":"sha256:e0dd0176e78461f6ae1db6226740b5abc1306f977695b116e137edb173ef5c72"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/bundle.json","state_url":"https://pith.science/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T11:16:35Z","links":{"resolver":"https://pith.science/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD","bundle":"https://pith.science/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/bundle.json","state":"https://pith.science/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/X7LZYKZRAUM6DO4YHMR6GAZAFD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:X7LZYKZRAUM6DO4YHMR6GAZAFD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"baf224a4f8b2127e2a4db580917a6c69cdff7e07be142e83ed269bc84922453e","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T04:12:01Z","title_canon_sha256":"68b723c5f2a3e099fef2cac0a7ba515a929448b49cd56c5b4adbae8c6dc40497"},"schema_version":"1.0","source":{"id":"2309.01940","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.01940","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"arxiv_version","alias_value":"2309.01940v4","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.01940","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_12","alias_value":"X7LZYKZRAUM6","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_16","alias_value":"X7LZYKZRAUM6DO4Y","created_at":"2026-07-05T07:54:32Z"},{"alias_kind":"pith_short_8","alias_value":"X7LZYKZR","created_at":"2026-07-05T07:54:32Z"}],"graph_snapshots":[{"event_id":"sha256:e0dd0176e78461f6ae1db6226740b5abc1306f977695b116e137edb173ef5c72","target":"graph","created_at":"2026-07-05T07:54:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.01940/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"With the emergence of Large Language Models (LLMs), there has been a significant improvement in the programming capabilities of models, attracting growing attention from researchers. Evaluating the programming capabilities of LLMs is crucial as it reflects the multifaceted abilities of LLMs, and it has numerous downstream applications. In this paper, we propose CodeApex, a bilingual benchmark dataset focusing on the programming comprehension, code generation, and code correction abilities of LLMs. Programming comprehension task tests LLMs on multiple-choice exam questions covering conceptual u","authors_text":"Huacan Chai, Jianghao Lin, Jiayi Lei, Jingkuan Wang, Kangning Zhang, Kounianhua Du, Lingyue Fu, Longteng Fan, Renting Rui, Shuang Luo, Siyuan Qi, Weiming Zhang, Weinan Zhang, Yifan Liu, Yong Yu, Yuchen Fang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T04:12:01Z","title":"CodeApex: A Bilingual Programming Evaluation Benchmark for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.01940","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:90421286ea7956008eff326027425b9bfe13bbbcbf1f2bbad91f8536f44dffbd","target":"record","created_at":"2026-07-05T07:54:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"baf224a4f8b2127e2a4db580917a6c69cdff7e07be142e83ed269bc84922453e","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T04:12:01Z","title_canon_sha256":"68b723c5f2a3e099fef2cac0a7ba515a929448b49cd56c5b4adbae8c6dc40497"},"schema_version":"1.0","source":{"id":"2309.01940","kind":"arxiv","version":4}},"canonical_sha256":"bfd79c2b310519e1bb983b23e3032028ee4052e44926f557fa5c768ee10fc320","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bfd79c2b310519e1bb983b23e3032028ee4052e44926f557fa5c768ee10fc320","first_computed_at":"2026-07-05T07:54:32.627203Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:54:32.627203Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"csNfxx0c/aekVhP60IZR4A7ibhUGO26yhhvacsiWgbvINNiuxXCo98A0jYU+3bSn3KHbg6niGNReRLwKNUFGAg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:54:32.627794Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.01940","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:90421286ea7956008eff326027425b9bfe13bbbcbf1f2bbad91f8536f44dffbd","sha256:e0dd0176e78461f6ae1db6226740b5abc1306f977695b116e137edb173ef5c72"],"state_sha256":"d063e79a093636bc537fc3782e111b154f8271a040538082f7c3da28ea8fae0b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7FrjUOWbuukauJxXBkDlJWv5GuOG3d39GsWwscPPBaKp5H05xALpKsC4ks/9ztSp+716/E6N64Sao6Vbg+RHCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T11:16:35.941621Z","bundle_sha256":"98fd5c99249089a4d10568a09d08ff56defdf5b48f24c34339d7d1dd658fab70"}}