{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:WRO4JY3BPFJ7D7OQWXMVCRJ3G2","short_pith_number":"pith:WRO4JY3B","canonical_record":{"source":{"id":"2405.11430","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","cross_cats_sorted":[],"title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b","abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44"},"schema_version":"1.0"},"canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","source":{"kind":"arxiv","id":"2405.11430","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"arxiv_version","alias_value":"2405.11430v3","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_12","alias_value":"WRO4JY3BPFJ7","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_16","alias_value":"WRO4JY3BPFJ7D7OQ","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_8","alias_value":"WRO4JY3B","created_at":"2026-07-05T11:54:39Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:WRO4JY3BPFJ7D7OQWXMVCRJ3G2","target":"record","payload":{"canonical_record":{"source":{"id":"2405.11430","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","cross_cats_sorted":[],"title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b","abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44"},"schema_version":"1.0"},"canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:54:39.720396Z","signature_b64":"A7lAFaOiNgPWtdt121A2O9hT+/tnxw6YIBko/vD79RsvcyG48eO1uwlAaKLGUfTrpW84BP1fwhDWLb9WgMPPBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","last_reissued_at":"2026-07-05T11:54:39.719842Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:54:39.719842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.11430","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:39Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GjwmBejhX2UL64EAyibwLf0SfODAeA8396YH6K5JnScr9pIn/2VYIx3kseWUZVeQUVDswriS5ncZJojfYlnRAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:58:54.081691Z"},"content_sha256":"73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c","schema_version":"1.0","event_id":"sha256:73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:WRO4JY3BPFJ7D7OQWXMVCRJ3G2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MHPP: Exploring the Capabilities and Limitations of Language Models Beyond Basic Code Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dong Huang, Guangtao Zeng, Haochen Tan, Jianbo Dai, Jianqiao Lu, Ming Cheng, Rongju Ruan, Yunlong Feng, Zhijiang Guo","submitted_at":"2024-05-19T03:08:02Z","abstract_excerpt":"Recent advancements in large language models (LLMs) have greatly improved code generation, specifically at the function level. For instance, GPT-4o has achieved a 91.0\\% pass rate on HumanEval. However, this draws into question the adequacy of existing benchmarks in thoroughly assessing function-level code generation capabilities. Our study analyzed two common benchmarks, HumanEval and MBPP, and found that these might not thoroughly evaluate LLMs' code generation capacities due to limitations in quality, difficulty, and granularity. To resolve this, we introduce the Mostly Hard Python Problems"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.11430","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.11430/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:39Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QJy7rIY5Vao5DxQVLoFEtLFlKdWYB4KyKVip+Sm/cheQMx9VM22Jr9Y5pAwZ8UkPyXBKaMVtpx5Cx9deYVx2Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:58:54.082391Z"},"content_sha256":"fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825","schema_version":"1.0","event_id":"sha256:fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/bundle.json","state_url":"https://pith.science/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T17:58:54Z","links":{"resolver":"https://pith.science/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2","bundle":"https://pith.science/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/bundle.json","state":"https://pith.science/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WRO4JY3BPFJ7D7OQWXMVCRJ3G2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WRO4JY3BPFJ7D7OQWXMVCRJ3G2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b"},"schema_version":"1.0","source":{"id":"2405.11430","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"arxiv_version","alias_value":"2405.11430v3","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.11430","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_12","alias_value":"WRO4JY3BPFJ7","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_16","alias_value":"WRO4JY3BPFJ7D7OQ","created_at":"2026-07-05T11:54:39Z"},{"alias_kind":"pith_short_8","alias_value":"WRO4JY3B","created_at":"2026-07-05T11:54:39Z"}],"graph_snapshots":[{"event_id":"sha256:fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825","target":"graph","created_at":"2026-07-05T11:54:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.11430/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advancements in large language models (LLMs) have greatly improved code generation, specifically at the function level. For instance, GPT-4o has achieved a 91.0\\% pass rate on HumanEval. However, this draws into question the adequacy of existing benchmarks in thoroughly assessing function-level code generation capabilities. Our study analyzed two common benchmarks, HumanEval and MBPP, and found that these might not thoroughly evaluate LLMs' code generation capacities due to limitations in quality, difficulty, and granularity. To resolve this, we introduce the Mostly Hard Python Problems","authors_text":"Dong Huang, Guangtao Zeng, Haochen Tan, Jianbo Dai, Jianqiao Lu, Ming Cheng, Rongju Ruan, Yunlong Feng, Zhijiang Guo","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title":"MHPP: Exploring the Capabilities and Limitations of Language Models Beyond Basic Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.11430","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c","target":"record","created_at":"2026-07-05T11:54:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9ff860e3a1092e300befa7003038c2d4f8412446a964316724057553ebac1d44","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-19T03:08:02Z","title_canon_sha256":"d01db30ac6ac830553eb56bfc075a824ee59189178e472a17243365be653205b"},"schema_version":"1.0","source":{"id":"2405.11430","kind":"arxiv","version":3}},"canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b45dc4e3617953f1fdd0b5d951453b36acfc007ece3617580b7f089ab9b28798","first_computed_at":"2026-07-05T11:54:39.719842Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:54:39.719842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A7lAFaOiNgPWtdt121A2O9hT+/tnxw6YIBko/vD79RsvcyG48eO1uwlAaKLGUfTrpW84BP1fwhDWLb9WgMPPBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:54:39.720396Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.11430","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:73cc4b288d5239bd6ab56bb6779a286c0d13fabe57e410cf5800d424c1962a7c","sha256:fddbf01b92716de113a54409b08ad556b57de0face49c4e52f448ab4215cd825"],"state_sha256":"a8b09cda1e1f4a392f85af8dafe2740b7059e11492d8a75f590815eab6f62357"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"E8YQUbHFaqXNHsn+StUnN/mx82YkkDaagAWlYwXU9b46jQ+iefHj8pbOSaEUJ2H1vl7NnXsuI/PLSHiOgwVJDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T17:58:54.087776Z","bundle_sha256":"ab8964ffece071cbe3d3c25f1381548f76404afcd6120029322e942486d513ed"}}