{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WRO4JY3BPFJ7D7OQWXMVCRJ3G2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b"},"schema_version":"1.0","source":{"id":"2405.11430","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"arxiv_version","alias_value":"2405.11430v3","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_12","alias_value":"WRO4JY3BPFJ7","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_16","alias_value":"WRO4JY3BPFJ7D7OQ","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_8","alias_value":"WRO4JY3B","created_at":"2026-07-05T11:54:39Z"}],"graph_snapshots":[{"event_id":"sha256:fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825","target":"graph","created_at":"2026-07-05T11:54:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.11430/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advancements in large language models (LLMs) have greatly improved code generation, specifically at the function level. For instance, GPT-4o has achieved a 91.0\\% pass rate on HumanEval. However, this draws into question the adequacy of existing benchmarks in thoroughly assessing function-level code generation capabilities. Our study analyzed two common benchmarks, HumanEval and MBPP, and found that these might not thoroughly evaluate LLMs' code generation capacities due to limitations in quality, difficulty, and granularity. To resolve this, we introduce the Mostly Hard Python Problems","authors_text":"Dong Huang, Guangtao Zeng, Haochen Tan, Jianbo Dai, Jianqiao Lu, Ming Cheng, Rongju Ruan, Yunlong Feng, Zhijiang Guo","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title":"MHPP: Exploring the Capabilities and Limitations of Language Models Beyond Basic Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.11430","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c","target":"record","created_at":"2026-07-05T11:54:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b"},"schema_version":"1.0","source":{"id":"2405.11430","kind":"arxiv","version":3}},"canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","first_computed_at":"2026-07-05T11:54:39.719842Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:54:39.719842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A7lAFaOiNgPWtdt121A2O9hT+/tnxw6YIBko/vD79RsvcyG48eO1uwlAaKLGUfTrpW84BP1fwhDWLb9WgMPPBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:54:39.720396Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.11430","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c","sha256:fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825"],"state_sha256":"a8b09cda1e1f4a392f85af8dafe2740b7059e11492d8a75f590815eab6f62357"}