{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:K5OGNZKWWGUNKWM4CPEYDERSCP","short_pith_number":"pith:K5OGNZKW","schema_version":"1.0","canonical_sha256":"575c66e556b1a8d5599c13c981923213fcbc63a192ac297ed5d3af25212ac48c","source":{"kind":"arxiv","id":"2408.06450","version":1},"attestation_state":"computed","paper":{"title":"Evaluating Language Models for Efficient Code Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Jiawei Liu, Junhao Wang, Lingming Zhang, Songrun Xie, Yifeng Ding, Yuxiang Wei","submitted_at":"2024-08-12T18:59:13Z","abstract_excerpt":"We introduce Differential Performance Evaluation (DPE), a framework designed to reliably evaluate Large Language Models (LLMs) for efficient code generation. Traditional coding benchmarks often fail to provide reliable insights into code efficiency, due to their reliance on simplistic test inputs and the absence of effective compound metrics. DPE addresses these issues by focusing on efficiency-demanding programming tasks and establishing an insightful compound metric for performance evaluation. DPE operates in two phases: To curate efficiency datasets, it selects efficiency-demanding tasks fr"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.06450","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-08-12T18:59:13Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"1796a2f887fa1dfcd8b5ea1731e07bc0791a856dc833d07ad82dcb364ad96e23","abstract_canon_sha256":"de21c11f26167addc7610268ca60b4f4dceff29c4319b9c0ce8fa97fa5a7201a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:54:57.045483Z","signature_b64":"GCmJpEaVKpJtQWILhaqjGm3wjVjBZs6DvdGSY/pB8WsclFvSZwEHzh6Yz2C+hqVEN9uKD5VXthWJ+lwmllPjCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"575c66e556b1a8d5599c13c981923213fcbc63a192ac297ed5d3af25212ac48c","last_reissued_at":"2026-07-05T08:54:57.044989Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:54:57.044989Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Evaluating Language Models for Efficient Code Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Jiawei Liu, Junhao Wang, Lingming Zhang, Songrun Xie, Yifeng Ding, Yuxiang Wei","submitted_at":"2024-08-12T18:59:13Z","abstract_excerpt":"We introduce Differential Performance Evaluation (DPE), a framework designed to reliably evaluate Large Language Models (LLMs) for efficient code generation. Traditional coding benchmarks often fail to provide reliable insights into code efficiency, due to their reliance on simplistic test inputs and the absence of effective compound metrics. DPE addresses these issues by focusing on efficiency-demanding programming tasks and establishing an insightful compound metric for performance evaluation. DPE operates in two phases: To curate efficiency datasets, it selects efficiency-demanding tasks fr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.06450","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.06450/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.06450","created_at":"2026-07-05T08:54:57.045043+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.06450v1","created_at":"2026-07-05T08:54:57.045043+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.06450","created_at":"2026-07-05T08:54:57.045043+00:00"},{"alias_kind":"pith_short_12","alias_value":"K5OGNZKWWGUN","created_at":"2026-07-05T08:54:57.045043+00:00"},{"alias_kind":"pith_short_16","alias_value":"K5OGNZKWWGUNKWM4","created_at":"2026-07-05T08:54:57.045043+00:00"},{"alias_kind":"pith_short_8","alias_value":"K5OGNZKW","created_at":"2026-07-05T08:54:57.045043+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07619","citing_title":"Rethinking Code Performance Benchmarks for LLMs","ref_index":99,"is_internal_anchor":true},{"citing_arxiv_id":"2606.06826","citing_title":"SkelDPO: A Skeleton-Guided Direct Preference Optimization Framework for Efficient Code Generation","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06821","citing_title":"Chiseling Out Efficiency: Structured Skeleton Supervision for Efficient Code Generation","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31767","citing_title":"JETO-Bench: A Reproducible Benchmark for Execution Time Improvement Patches in Java","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2605.26842","citing_title":"MONA: Muon Optimizer with Nesterov Acceleration for Scalable Language Model Training","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27051","citing_title":"ConVer: Using Contracts and Loop Invariant Synthesis for Scalable Formal Software Verification","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2506.01249","citing_title":"SysLLMatic: Large Language Models are Software System Optimizers","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2510.15494","citing_title":"Do AI Models Dream of Faster Code? An Empirical Study on LLM-Proposed Performance Improvements in Real-World Software","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10890","citing_title":"CppPerf: An Automated Pipeline and Dataset for Performance-Improving C++ Commits","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP","json":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP.json","graph_json":"https://pith.science/api/pith-number/K5OGNZKWWGUNKWM4CPEYDERSCP/graph.json","events_json":"https://pith.science/api/pith-number/K5OGNZKWWGUNKWM4CPEYDERSCP/events.json","paper":"https://pith.science/paper/K5OGNZKW"},"agent_actions":{"view_html":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP","download_json":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP.json","view_paper":"https://pith.science/paper/K5OGNZKW","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.06450&json=true","fetch_graph":"https://pith.science/api/pith-number/K5OGNZKWWGUNKWM4CPEYDERSCP/graph.json","fetch_events":"https://pith.science/api/pith-number/K5OGNZKWWGUNKWM4CPEYDERSCP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP/action/storage_attestation","attest_author":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP/action/author_attestation","sign_citation":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP/action/citation_signature","submit_replication":"https://pith.science/pith/K5OGNZKWWGUNKWM4CPEYDERSCP/action/replication_record"}},"created_at":"2026-07-05T08:54:57.045043+00:00","updated_at":"2026-07-05T08:54:57.045043+00:00"}