{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YUR3O2VFCQMDG536PNBDWOGHTZ","short_pith_number":"pith:YUR3O2VF","canonical_record":{"source":{"id":"2405.19856","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T09:03:42Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"615bdcd0a2d9ac90aed634ba9466aef7ebf456aa7f1a7e0dd0a589f9b13f3f95","abstract_canon_sha256":"3e3b7fd53fa057353bf6944f062036120934c832d29d87a9924c7a0d3f1d8451"},"schema_version":"1.0"},"canonical_sha256":"c523b76aa5141833777e7b423b38c79e7a2e855e4b53bbb4052d17d475356a3f","source":{"kind":"arxiv","id":"2405.19856","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19856","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19856v1","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19856","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_12","alias_value":"YUR3O2VFCQMD","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_16","alias_value":"YUR3O2VFCQMDG536","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_8","alias_value":"YUR3O2VF","created_at":"2026-07-05T08:25:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YUR3O2VFCQMDG536PNBDWOGHTZ","target":"record","payload":{"canonical_record":{"source":{"id":"2405.19856","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T09:03:42Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"615bdcd0a2d9ac90aed634ba9466aef7ebf456aa7f1a7e0dd0a589f9b13f3f95","abstract_canon_sha256":"3e3b7fd53fa057353bf6944f062036120934c832d29d87a9924c7a0d3f1d8451"},"schema_version":"1.0"},"canonical_sha256":"c523b76aa5141833777e7b423b38c79e7a2e855e4b53bbb4052d17d475356a3f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:25:13.490064Z","signature_b64":"awm7b3dqKcMXwsEi81GzXsYTUvsSxsTMnJQD6cZx9rH7Vmb9mjNziYPRjWGltyGlY5jTSe8VzLTWi7iW8g0IBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c523b76aa5141833777e7b423b38c79e7a2e855e4b53bbb4052d17d475356a3f","last_reissued_at":"2026-07-05T08:25:13.489578Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:25:13.489578Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.19856","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:25:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YyIROA6nIg/UimDfRterqf0BZ8XsuDG57tUt1D8+wnjyTtDJXU4eUZuBuiSuuwvx0X2s/uGJwOenJ93trGjcBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:48:44.058935Z"},"content_sha256":"62ea1347359102f482f5853520bbec46b5c192e36022d9c55a2cbcdc9316d7d4","schema_version":"1.0","event_id":"sha256:62ea1347359102f482f5853520bbec46b5c192e36022d9c55a2cbcdc9316d7d4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YUR3O2VFCQMDG536PNBDWOGHTZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.SE"],"primary_cat":"cs.CL","authors_text":"Binhua Li, Fei Huang, Ge Li, Hao Zhu, Huanyu Liu, Jia Li, Jiazheng Ding, Kaibo Liu, Lanshen Wang, Lecheng Wang, Xuanming Zhang, Yihong Dong, Yongbin Li, Yongmin Li, Yunfei Zhao, Yuqi Zhu, Zheng Fang, Zhi Jin","submitted_at":"2024-05-30T09:03:42Z","abstract_excerpt":"How to evaluate the coding abilities of Large Language Models (LLMs) remains an open question. We find that existing benchmarks are poorly aligned with real-world code repositories and are insufficient to evaluate the coding abilities of LLMs.\n  To address the knowledge gap, we propose a new benchmark named DevEval, which has three advances. (1) DevEval aligns with real-world repositories in multiple dimensions, e.g., code distributions and dependency distributions. (2) DevEval is annotated by 13 developers and contains comprehensive annotations (e.g., requirements, original repositories, refe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19856","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.19856/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:25:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"f9bt2aXvwjbIt5RRzNzmx/wfgWIeSmn5ySwHIatZbsf8Y/djkQBPuobgMCThQncmi5/U0s0avAIoB4ULlM+yBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:48:44.059307Z"},"content_sha256":"dd58478084c092a46679f20fc443ae3900c333df94aa603bc62037589067896d","schema_version":"1.0","event_id":"sha256:dd58478084c092a46679f20fc443ae3900c333df94aa603bc62037589067896d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/bundle.json","state_url":"https://pith.science/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T03:48:44Z","links":{"resolver":"https://pith.science/pith/YUR3O2VFCQMDG536PNBDWOGHTZ","bundle":"https://pith.science/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/bundle.json","state":"https://pith.science/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YUR3O2VFCQMDG536PNBDWOGHTZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YUR3O2VFCQMDG536PNBDWOGHTZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3e3b7fd53fa057353bf6944f062036120934c832d29d87a9924c7a0d3f1d8451","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T09:03:42Z","title_canon_sha256":"615bdcd0a2d9ac90aed634ba9466aef7ebf456aa7f1a7e0dd0a589f9b13f3f95"},"schema_version":"1.0","source":{"id":"2405.19856","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19856","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19856v1","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19856","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_12","alias_value":"YUR3O2VFCQMD","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_16","alias_value":"YUR3O2VFCQMDG536","created_at":"2026-07-05T08:25:13Z"},{"alias_kind":"pith_short_8","alias_value":"YUR3O2VF","created_at":"2026-07-05T08:25:13Z"}],"graph_snapshots":[{"event_id":"sha256:dd58478084c092a46679f20fc443ae3900c333df94aa603bc62037589067896d","target":"graph","created_at":"2026-07-05T08:25:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.19856/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"How to evaluate the coding abilities of Large Language Models (LLMs) remains an open question. We find that existing benchmarks are poorly aligned with real-world code repositories and are insufficient to evaluate the coding abilities of LLMs.\n  To address the knowledge gap, we propose a new benchmark named DevEval, which has three advances. (1) DevEval aligns with real-world repositories in multiple dimensions, e.g., code distributions and dependency distributions. (2) DevEval is annotated by 13 developers and contains comprehensive annotations (e.g., requirements, original repositories, refe","authors_text":"Binhua Li, Fei Huang, Ge Li, Hao Zhu, Huanyu Liu, Jia Li, Jiazheng Ding, Kaibo Liu, Lanshen Wang, Lecheng Wang, Xuanming Zhang, Yihong Dong, Yongbin Li, Yongmin Li, Yunfei Zhao, Yuqi Zhu, Zheng Fang, Zhi Jin","cross_cats":["cs.SE"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T09:03:42Z","title":"DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19856","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:62ea1347359102f482f5853520bbec46b5c192e36022d9c55a2cbcdc9316d7d4","target":"record","created_at":"2026-07-05T08:25:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3e3b7fd53fa057353bf6944f062036120934c832d29d87a9924c7a0d3f1d8451","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T09:03:42Z","title_canon_sha256":"615bdcd0a2d9ac90aed634ba9466aef7ebf456aa7f1a7e0dd0a589f9b13f3f95"},"schema_version":"1.0","source":{"id":"2405.19856","kind":"arxiv","version":1}},"canonical_sha256":"c523b76aa5141833777e7b423b38c79e7a2e855e4b53bbb4052d17d475356a3f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c523b76aa5141833777e7b423b38c79e7a2e855e4b53bbb4052d17d475356a3f","first_computed_at":"2026-07-05T08:25:13.489578Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:25:13.489578Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"awm7b3dqKcMXwsEi81GzXsYTUvsSxsTMnJQD6cZx9rH7Vmb9mjNziYPRjWGltyGlY5jTSe8VzLTWi7iW8g0IBA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:25:13.490064Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.19856","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:62ea1347359102f482f5853520bbec46b5c192e36022d9c55a2cbcdc9316d7d4","sha256:dd58478084c092a46679f20fc443ae3900c333df94aa603bc62037589067896d"],"state_sha256":"1a1a1130eb7d3a32cbd76c21c321ed033e378b31a2e9bf454950ac4b29705835"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fHKGZz0QNkmGtGmqcDth0G9Hc8ie2Q1UcD4ACf4vpDr/4e5V5AzisOiAyl8BjWEY5nVlZwKIHmdZwE6PVRDZBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T03:48:44.061326Z","bundle_sha256":"d29cc30952f4d7b19ef3ed488bbc554f5bbf5449cab49f0b653226f0f67b39cf"}}