{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IO4ZESJ54XR7XFDTHQAB2FGLLU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b2ea03c31f057c670d4b3c3f7265ea05fc2bebd6ef557da8d56e4f222e49396f","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-02-12T17:53:22Z","title_canon_sha256":"f0750fef0643fd3dada7e794c5cb5272fdcaa7f93b744225465a529a02bf577c"},"schema_version":"1.0","source":{"id":"2402.07844","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.07844","created_at":"2026-07-05T08:30:08Z"},{"alias_kind":"arxiv_version","alias_value":"2402.07844v4","created_at":"2026-07-05T08:30:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.07844","created_at":"2026-07-05T08:30:08Z"},{"alias_kind":"pith_short_12","alias_value":"IO4ZESJ54XR7","created_at":"2026-07-05T08:30:08Z"},{"alias_kind":"pith_short_16","alias_value":"IO4ZESJ54XR7XFDT","created_at":"2026-07-05T08:30:08Z"},{"alias_kind":"pith_short_8","alias_value":"IO4ZESJ5","created_at":"2026-07-05T08:30:08Z"}],"graph_snapshots":[{"event_id":"sha256:268902ce826492768c92c5f8ee05010a1b1423535540afbfb32b8592ca2dafb7","target":"graph","created_at":"2026-07-05T08:30:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.07844/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Amidst the recent strides in evaluating Large Language Models for Code (Code LLMs), existing benchmarks have mainly focused on the functional correctness of generated code, neglecting the importance of their computational efficiency. To fill the gap, we present Mercury, the first code efficiency benchmark for Code LLMs. It comprises 1,889 Python tasks, each accompanied by adequate solutions that serve as real-world efficiency baselines, enabling a comprehensive analysis of the runtime distribution. Based on the distribution, we introduce a new metric Beyond, which computes a runtime-percentile","authors_text":"Anh Tuan Luu, Bin Ji, Mingzhe Du, Qian Liu, See-kiong Ng","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-02-12T17:53:22Z","title":"Mercury: A Code Efficiency Benchmark for Code Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.07844","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5830d47ff649fb7366989cce3aa911fa5c23e2f40bd5f905031233b5bcaf925f","target":"record","created_at":"2026-07-05T08:30:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b2ea03c31f057c670d4b3c3f7265ea05fc2bebd6ef557da8d56e4f222e49396f","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-02-12T17:53:22Z","title_canon_sha256":"f0750fef0643fd3dada7e794c5cb5272fdcaa7f93b744225465a529a02bf577c"},"schema_version":"1.0","source":{"id":"2402.07844","kind":"arxiv","version":4}},"canonical_sha256":"43b992493de5e3fb94733c001d14cb5d05894272107236254bfbbf1379894ce3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"43b992493de5e3fb94733c001d14cb5d05894272107236254bfbbf1379894ce3","first_computed_at":"2026-07-05T08:30:08.520711Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:30:08.520711Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"CYBpL1FBj71mlQLdpY5P0isiLowA8Xhiiv5veTWSwZ27ZVwPiOV1aviZIVCvNggP605HwMz9OiP/khsmHdWXDA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:30:08.521185Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.07844","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5830d47ff649fb7366989cce3aa911fa5c23e2f40bd5f905031233b5bcaf925f","sha256:268902ce826492768c92c5f8ee05010a1b1423535540afbfb32b8592ca2dafb7"],"state_sha256":"15e43feae32801c727d55cb6234dffe06ee5928bc4f2a1749d7f0f13c649c086"}