{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:DXOIU5VTEFEPZCHODNQ5RNYOG2","short_pith_number":"pith:DXOIU5VT","canonical_record":{"source":{"id":"2501.10711","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-01-18T09:51:57Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"0cc7a9eecf6bc0db96d6167e710f1bf6df3958a9de179fc95c3826f1381fd20b","abstract_canon_sha256":"7db5601ed86e13add943dad5d203c8bd59e182fea143bc291ca494df015ced74"},"schema_version":"1.0"},"canonical_sha256":"1ddc8a76b32148fc88ee1b61d8b70e368740a3c8196e216b97f215013e863ba9","source":{"kind":"arxiv","id":"2501.10711","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.10711","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"arxiv_version","alias_value":"2501.10711v5","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.10711","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_12","alias_value":"DXOIU5VTEFEP","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_16","alias_value":"DXOIU5VTEFEPZCHO","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_8","alias_value":"DXOIU5VT","created_at":"2026-07-07T02:19:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:DXOIU5VTEFEPZCHODNQ5RNYOG2","target":"record","payload":{"canonical_record":{"source":{"id":"2501.10711","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-01-18T09:51:57Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"0cc7a9eecf6bc0db96d6167e710f1bf6df3958a9de179fc95c3826f1381fd20b","abstract_canon_sha256":"7db5601ed86e13add943dad5d203c8bd59e182fea143bc291ca494df015ced74"},"schema_version":"1.0"},"canonical_sha256":"1ddc8a76b32148fc88ee1b61d8b70e368740a3c8196e216b97f215013e863ba9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-07T02:19:37.250235Z","signature_b64":"+AorFA71Sj21iFRhKEiD0CXa5BLm/CUhRaen7S/xy9hllAqTjLqGyXR6HjGyaqDLR4TYvyjZVgx8P9VCkPNADw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1ddc8a76b32148fc88ee1b61d8b70e368740a3c8196e216b97f215013e863ba9","last_reissued_at":"2026-07-07T02:19:37.249459Z","signature_status":"signed_v1","first_computed_at":"2026-07-07T02:19:37.249459Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2501.10711","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-07T02:19:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EOJh8ZRjzActEh5qNZ3eNvYscnDNF25KNxZsT3v/iv8aJvHqg5URw5ZMBJqYLpYMudKyXZOd1DXtmGZyI6suCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T02:25:55.618597Z"},"content_sha256":"5ee501356e1e67a27f1e76b95a2b65a2ce9c38b7f6a8202d57366376f3630324","schema_version":"1.0","event_id":"sha256:5ee501356e1e67a27f1e76b95a2b65a2ce9c38b7f6a8202d57366376f3630324"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:DXOIU5VTEFEPZCHODNQ5RNYOG2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.SE","authors_text":"Boxi Yu, Chaozheng Wang, Jialun Cao, Michael R. Lyu, Mingwei Liu, Pinjia He, Ruixi Qiao, Shing-Chi Cheung, Shuai Wang, Shuqing Li, Wenxuan Wang, Yuk-Kit Chan, Yuting Han, Zibin Zheng, Zixuan Ling","submitted_at":"2025-01-18T09:51:57Z","abstract_excerpt":"Code-related benchmarks play a critical role in evaluating large language models (LLMs), yet their quality fundamentally shapes how the community interprets model capabilities. In the past few years, awareness of benchmark quality has grown. Yet, after a decade-scale (2014-2025) survey over 672 code benchmarks, we observed a lag between growing awareness and actual practice. For example, in 2025 alone, the number of benchmarks that ignore code coverage when providing test cases nearly matches the total count accumulated across the previous ten years. In response, we take a clear position: Code"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.10711","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.10711/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-07T02:19:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zVyL/T+FDyKWDdyOSJUJlNaTqTSDBH+HxOJct650EcDHTnY7hVU3JW5jm94w//mzkLtJNEKylUY7U12Bjr5xCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T02:25:55.619117Z"},"content_sha256":"750d3cfe30daf2a99d4fc167ae919ec36dd4c9b990d345ab784c93ed0301e127","schema_version":"1.0","event_id":"sha256:750d3cfe30daf2a99d4fc167ae919ec36dd4c9b990d345ab784c93ed0301e127"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/bundle.json","state_url":"https://pith.science/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T02:25:55Z","links":{"resolver":"https://pith.science/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2","bundle":"https://pith.science/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/bundle.json","state":"https://pith.science/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DXOIU5VTEFEPZCHODNQ5RNYOG2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:DXOIU5VTEFEPZCHODNQ5RNYOG2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7db5601ed86e13add943dad5d203c8bd59e182fea143bc291ca494df015ced74","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-01-18T09:51:57Z","title_canon_sha256":"0cc7a9eecf6bc0db96d6167e710f1bf6df3958a9de179fc95c3826f1381fd20b"},"schema_version":"1.0","source":{"id":"2501.10711","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.10711","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"arxiv_version","alias_value":"2501.10711v5","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.10711","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_12","alias_value":"DXOIU5VTEFEP","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_16","alias_value":"DXOIU5VTEFEPZCHO","created_at":"2026-07-07T02:19:37Z"},{"alias_kind":"pith_short_8","alias_value":"DXOIU5VT","created_at":"2026-07-07T02:19:37Z"}],"graph_snapshots":[{"event_id":"sha256:750d3cfe30daf2a99d4fc167ae919ec36dd4c9b990d345ab784c93ed0301e127","target":"graph","created_at":"2026-07-07T02:19:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.10711/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Code-related benchmarks play a critical role in evaluating large language models (LLMs), yet their quality fundamentally shapes how the community interprets model capabilities. In the past few years, awareness of benchmark quality has grown. Yet, after a decade-scale (2014-2025) survey over 672 code benchmarks, we observed a lag between growing awareness and actual practice. For example, in 2025 alone, the number of benchmarks that ignore code coverage when providing test cases nearly matches the total count accumulated across the previous ten years. In response, we take a clear position: Code","authors_text":"Boxi Yu, Chaozheng Wang, Jialun Cao, Michael R. Lyu, Mingwei Liu, Pinjia He, Ruixi Qiao, Shing-Chi Cheung, Shuai Wang, Shuqing Li, Wenxuan Wang, Yuk-Kit Chan, Yuting Han, Zibin Zheng, Zixuan Ling","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.10711","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5ee501356e1e67a27f1e76b95a2b65a2ce9c38b7f6a8202d57366376f3630324","target":"record","created_at":"2026-07-07T02:19:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7db5601ed86e13add943dad5d203c8bd59e182fea143bc291ca494df015ced74","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-01-18T09:51:57Z","title_canon_sha256":"0cc7a9eecf6bc0db96d6167e710f1bf6df3958a9de179fc95c3826f1381fd20b"},"schema_version":"1.0","source":{"id":"2501.10711","kind":"arxiv","version":5}},"canonical_sha256":"1ddc8a76b32148fc88ee1b61d8b70e368740a3c8196e216b97f215013e863ba9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1ddc8a76b32148fc88ee1b61d8b70e368740a3c8196e216b97f215013e863ba9","first_computed_at":"2026-07-07T02:19:37.249459Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-07T02:19:37.249459Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+AorFA71Sj21iFRhKEiD0CXa5BLm/CUhRaen7S/xy9hllAqTjLqGyXR6HjGyaqDLR4TYvyjZVgx8P9VCkPNADw==","signature_status":"signed_v1","signed_at":"2026-07-07T02:19:37.250235Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.10711","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5ee501356e1e67a27f1e76b95a2b65a2ce9c38b7f6a8202d57366376f3630324","sha256:750d3cfe30daf2a99d4fc167ae919ec36dd4c9b990d345ab784c93ed0301e127"],"state_sha256":"d17aab7a69441f307ea044e92304da52d76debc53482cee3c454ef6df14f8660"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aJUgn4GocnoyzQUui98exArhqXcCs/5IN43eA3xz5sVnTnlAhqL7dwqPP/nFRIdQxrA2yfDD9DdU/m/l5Ah8Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T02:25:55.624322Z","bundle_sha256":"e00e62db0c911b83375bf7311c9a53de3a0fe36cdff51ce059ef8854bb18f71a"}}