{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:AARY37WBSXTPWMW5TVTXEYHQQX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3379dd3feb7dc3a4a4c597fdbe4e5579af83915fd86765d7a61389721213c942","cross_cats_sorted":["cs.CL","cs.PL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2026-05-13T08:10:26Z","title_canon_sha256":"73db000b4f0a474b1bde5d52111578c40706987ac468084b07681e8f9b247a65"},"schema_version":"1.0","source":{"id":"2605.15222","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.15222","created_at":"2026-05-20T00:00:47Z"},{"alias_kind":"arxiv_version","alias_value":"2605.15222v1","created_at":"2026-05-20T00:00:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.15222","created_at":"2026-05-20T00:00:47Z"},{"alias_kind":"pith_short_12","alias_value":"AARY37WBSXTP","created_at":"2026-05-20T00:00:47Z"},{"alias_kind":"pith_short_16","alias_value":"AARY37WBSXTPWMW5","created_at":"2026-05-20T00:00:47Z"},{"alias_kind":"pith_short_8","alias_value":"AARY37WB","created_at":"2026-05-20T00:00:47Z"}],"graph_snapshots":[{"event_id":"sha256:7e7c7ae9e53cc63b7554f1f966bfcf3ceea95747841aba1f906cf68ff5a88839","target":"graph","created_at":"2026-05-20T00:00:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Our evaluation on a broad set of state-of-the-art LLMs shows a clear gap between model-generated code and expert-optimized implementations. The gap is especially large on tasks involving parallelism and GPU operations."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The selected tasks accurately capture realistic system-level implementation choices, hardware-aware optimizations, and performance bottlenecks that matter in practice."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"PerfCodeBench reveals that state-of-the-art LLMs produce functionally correct but significantly slower code than expert-optimized versions on system-level tasks, especially those involving parallelism and GPUs."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Current LLMs produce code that is functionally correct but far from expert-optimized on system-level performance tasks."}],"snapshot_sha256":"ce9f3414af28d8893fc1a2f1c87a7f262d6036acd0ada58b77999e174696a3b1"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[{"findings_count":0,"name":"claim_evidence","ran_at":"2026-05-19T20:21:57.158243Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_title_agreement","ran_at":"2026-05-19T18:31:18.819737Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_compliance","ran_at":"2026-05-19T18:10:16.387371Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"ai_meta_artifact","ran_at":"2026-05-19T13:33:22.835500Z","status":"skipped","version":"1.0.0"}],"endpoint":"/pith/2605.15222/integrity.json","findings":[],"snapshot_sha256":"17bae781a0bec8d0c469624434c6193720fefc03dae918aa25262292e1530d19","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) can often generate functionally correct code, but their ability to produce efficient implementations for performance-critical systems tasks remains limited. Existing code benchmarks mainly emphasize correctness or algorithmic problem solving, while realistic systems-level optimization is still underexplored. To address this gap, we introduce PerfCodeBench, an executable benchmark for evaluating LLMs on high-performance code optimization. The tasks require system-level implementation choices, hardware-aware optimization, and careful handling of performance bottlenec","authors_text":"Hanyu Yang, Haochen Shi, Haoran Li, Huihao Jing, Shaojin Chen, Sirui Zhang, Wenbin Hu, Yangqiu Song","cross_cats":["cs.CL","cs.PL"],"headline":"Current LLMs produce code that is functionally correct but far from expert-optimized on system-level performance tasks.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2026-05-13T08:10:26Z","title":"PerfCodeBench: Benchmarking LLMs for System-Level High-Performance Code Optimization"},"references":{"count":52,"internal_anchors":4,"resolved_work":52,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Introducing Claude Opus 4.5","work_id":"785c69cd-ef52-4c1c-80f5-3c8645ec6dbc","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Claude Code Overview","work_id":"dcfdde82-d743-457f-b8f4-1f23552dc365","year":2026},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Claude model overview","work_id":"114f4f5b-3359-458c-ba85-d8e90a92db5e","year":2026},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Understanding software engineering agents: A study of thought-action-result trajectories","work_id":"ddc723de-4578-4bab-a626-5da26d2d16f7","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"ByteDance Seed. Seed2.0.https://seed.bytedance.com/en/seed2, 2026","work_id":"d73a7341-aa04-4bbf-8412-53ceacca5313","year":2026}],"snapshot_sha256":"948508292d948b51f13bfef70f9292e6f97b56534db1d54f33a298acfb604e6c"},"source":{"id":"2605.15222","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-19T17:58:50.521493Z","id":"56bb79b6-e790-4b12-8c5b-d5dd3b17467a","model_set":{"reader":"grok-4.3"},"one_line_summary":"PerfCodeBench reveals that state-of-the-art LLMs produce functionally correct but significantly slower code than expert-optimized versions on system-level tasks, especially those involving parallelism and GPUs.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Current LLMs produce code that is functionally correct but far from expert-optimized on system-level performance tasks.","strongest_claim":"Our evaluation on a broad set of state-of-the-art LLMs shows a clear gap between model-generated code and expert-optimized implementations. The gap is especially large on tasks involving parallelism and GPU operations.","weakest_assumption":"The selected tasks accurately capture realistic system-level implementation choices, hardware-aware optimizations, and performance bottlenecks that matter in practice."}},"verdict_id":"56bb79b6-e790-4b12-8c5b-d5dd3b17467a"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8a3d8450ea171d085bbdc09d81c793d1f8c4150a6bf6d2108145e1c038458258","target":"record","created_at":"2026-05-20T00:00:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3379dd3feb7dc3a4a4c597fdbe4e5579af83915fd86765d7a61389721213c942","cross_cats_sorted":["cs.CL","cs.PL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2026-05-13T08:10:26Z","title_canon_sha256":"73db000b4f0a474b1bde5d52111578c40706987ac468084b07681e8f9b247a65"},"schema_version":"1.0","source":{"id":"2605.15222","kind":"arxiv","version":1}},"canonical_sha256":"00238dfec195e6fb32dd9d677260f085f8cad29fd1b6df629cba53839b9cfbfb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"00238dfec195e6fb32dd9d677260f085f8cad29fd1b6df629cba53839b9cfbfb","first_computed_at":"2026-05-20T00:00:47.041876Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-20T00:00:47.041876Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cy55gW0C26Ew4bTQWYhBzoeuDSd79DUB069588IFkX+fUy/ddjuok+UHyU3JaQQN3zqrK18cvBUDJPr5P7KhDg==","signature_status":"signed_v1","signed_at":"2026-05-20T00:00:47.042573Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.15222","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8a3d8450ea171d085bbdc09d81c793d1f8c4150a6bf6d2108145e1c038458258","sha256:7e7c7ae9e53cc63b7554f1f966bfcf3ceea95747841aba1f906cf68ff5a88839"],"state_sha256":"2f146a0fb5173b14e7744f9afc2e4a8fbc09dcc180cb0f95afc3e5517a86a7fe"}