{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:YUT446WGYGCUJE3OQOU4RME3XY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"97f573f3e96352cc348413b40f4e01ababdc4f6273e9c61361eba85776996e27","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-03-10T07:47:27Z","title_canon_sha256":"f310c487949579255a0033ffca81cc842813738558ab830db0d33c2cefadc2fa"},"schema_version":"1.0","source":{"id":"2503.07010","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.07010","created_at":"2026-07-05T11:13:22Z"},{"alias_kind":"arxiv_version","alias_value":"2503.07010v2","created_at":"2026-07-05T11:13:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.07010","created_at":"2026-07-05T11:13:22Z"},{"alias_kind":"pith_short_12","alias_value":"YUT446WGYGCU","created_at":"2026-07-05T11:13:22Z"},{"alias_kind":"pith_short_16","alias_value":"YUT446WGYGCUJE3O","created_at":"2026-07-05T11:13:22Z"},{"alias_kind":"pith_short_8","alias_value":"YUT446WG","created_at":"2026-07-05T11:13:22Z"}],"graph_snapshots":[{"event_id":"sha256:63ac2ac8badf7e736239268ecb9302ebe788fbe3c269cf1e5c28d9fd81d2b77c","target":"graph","created_at":"2026-07-05T11:13:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.07010/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recently, LLM agents have made rapid progress in improving their programming capabilities. However, existing benchmarks lack the ability to automatically evaluate from users' perspective, and also lack the explainability of the results of LLM agents' code generation capabilities. Thus, we introduce ProjectEval, a new benchmark for LLM agents project-level code generation's automated evaluation by simulating user interaction. ProjectEval is constructed by LLM with human reviewing. It has three different level inputs of natural languages or code skeletons. ProjectEval can evaluate the generated ","authors_text":"Daojing He, Jing Li, Kaiyuan Liu, Tianrun Gao, Yang Xiang, Yexing Du, Youcheng Pan","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-03-10T07:47:27Z","title":"ProjectEval: A Benchmark for Programming Agents Automated Evaluation on Project-Level Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.07010","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6241fb221a1ac9907fd3d54cd67f712c92c62e09ccc7956dcd0f2860566cab3f","target":"record","created_at":"2026-07-05T11:13:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"97f573f3e96352cc348413b40f4e01ababdc4f6273e9c61361eba85776996e27","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-03-10T07:47:27Z","title_canon_sha256":"f310c487949579255a0033ffca81cc842813738558ab830db0d33c2cefadc2fa"},"schema_version":"1.0","source":{"id":"2503.07010","kind":"arxiv","version":2}},"canonical_sha256":"c527ce7ac6c18544936e83a9c8b09bbe2b21125dd7cdc7a4561a8dc43b53cbc6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c527ce7ac6c18544936e83a9c8b09bbe2b21125dd7cdc7a4561a8dc43b53cbc6","first_computed_at":"2026-07-05T11:13:22.337206Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:13:22.337206Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"QJ8B2erQ362m1mwuZxOCDhEyLCaOTw01rvRG3UcntEpEvBTTZDg9YVeLycee8XEMLPmKr/SGlatBLrCuI2w3Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:13:22.337691Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.07010","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6241fb221a1ac9907fd3d54cd67f712c92c62e09ccc7956dcd0f2860566cab3f","sha256:63ac2ac8badf7e736239268ecb9302ebe788fbe3c269cf1e5c28d9fd81d2b77c"],"state_sha256":"86193ea71f0b9ec96699f57e2d8ff2eaaf29829c1561a41ea72d0de38d02d45a"}