{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:Q6JYRBA52K2D2R3IBPLEI6NOCW","short_pith_number":"pith:Q6JYRBA5","canonical_record":{"source":{"id":"2605.09059","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-05-09T17:05:44Z","cross_cats_sorted":[],"title_canon_sha256":"7a1170a886a839d58bc029648a6f567746c011c4ad1a7d8271d38087b9aa7b5e","abstract_canon_sha256":"4b89152d2063d8e7594c94602a53c3c7b9a3d6b5ead2bc613d34e5dc4121b420"},"schema_version":"1.0"},"canonical_sha256":"879388841dd2b43d47680bd64479ae15b5d9d59df3640a661e05d4f6c921f58c","source":{"kind":"arxiv","id":"2605.09059","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.09059","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"arxiv_version","alias_value":"2605.09059v2","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.09059","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_12","alias_value":"Q6JYRBA52K2D","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_16","alias_value":"Q6JYRBA52K2D2R3I","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_8","alias_value":"Q6JYRBA5","created_at":"2026-06-11T00:08:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:Q6JYRBA52K2D2R3IBPLEI6NOCW","target":"record","payload":{"canonical_record":{"source":{"id":"2605.09059","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-05-09T17:05:44Z","cross_cats_sorted":[],"title_canon_sha256":"7a1170a886a839d58bc029648a6f567746c011c4ad1a7d8271d38087b9aa7b5e","abstract_canon_sha256":"4b89152d2063d8e7594c94602a53c3c7b9a3d6b5ead2bc613d34e5dc4121b420"},"schema_version":"1.0"},"canonical_sha256":"879388841dd2b43d47680bd64479ae15b5d9d59df3640a661e05d4f6c921f58c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-11T00:08:12.679456Z","signature_b64":"6gmTfvkoxaWF6dPK4WozeHjo7Df45rzw9F2tdsSq43jmDcujsUwBCn3y/DtPVtn2yOPgDDFO1czNpwSZHso1Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"879388841dd2b43d47680bd64479ae15b5d9d59df3640a661e05d4f6c921f58c","last_reissued_at":"2026-06-11T00:08:12.678488Z","signature_status":"signed_v1","first_computed_at":"2026-06-11T00:08:12.678488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2605.09059","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-11T00:08:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pwt6q4uW+GJUPiuHZVR3BQnN5noq2HHUQYEr8VL684VTqp8zWnP/ILjyWrtCS5fMFwE7k+0B923B7B/FLDuaDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T14:14:35.784966Z"},"content_sha256":"aa04f004931682ff397777a05e83d5df579cd3120a0710340383c13157128559","schema_version":"1.0","event_id":"sha256:aa04f004931682ff397777a05e83d5df579cd3120a0710340383c13157128559"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:Q6JYRBA52K2D2R3IBPLEI6NOCW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating LLM-Generated Code: A Benchmark and Developer Study","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook.","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Anne Schwerk, Joanna Szych","submitted_at":"2026-05-09T17:05:44Z","abstract_excerpt":"Code generation is one of the tasks for which the use of Large Language Models is widely adopted and highly successful. Given this popularity, there are many benchmarks dedicated to code generation that can help select the best model. However, they primarily focus on measuring solution correctness, leaving other aspects, such as code quality and usability, behind. This paper aims to describe a custom tree-fold evaluation methodology for code generated by Large Language Models that bridges this gap. The methodology includes a dedicated correctness benchmark based on a complex multi-level comput"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"reviews gathered from developers can yield many new findings, especially those related to the code being in a production-ready state, that would not be possible to obtain using the standard correctness-focused benchmark approach.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That a structured code-review process with (unspecified number of) developers produces generalizable, unbiased insights into production readiness that standard benchmarks miss.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"A custom three-fold methodology combining a complex-project correctness benchmark, code quality verification, and structured developer reviews to evaluate LLM-generated code beyond correctness alone.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"4f1c40dc50666d95f215cfc0d18734115166fcba4cb135eeefa53f28bf833bcc"},"source":{"id":"2605.09059","kind":"arxiv","version":2},"verdict":{"id":"168bddb9-722f-4e2d-8eb3-aeaae40c2f37","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-12T02:22:34.681743Z","strongest_claim":"reviews gathered from developers can yield many new findings, especially those related to the code being in a production-ready state, that would not be possible to obtain using the standard correctness-focused benchmark approach.","one_line_summary":"A custom three-fold methodology combining a complex-project correctness benchmark, code quality verification, and structured developer reviews to evaluate LLM-generated code beyond correctness alone.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That a structured code-review process with (unspecified number of) developers produces generalizable, unbiased insights into production readiness that standard benchmarks miss.","pith_extraction_headline":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2605.09059/integrity.json","findings":[],"available":true,"detectors_run":[{"name":"claim_evidence","ran_at":"2026-05-20T08:22:01.706986Z","status":"completed","version":"1.0.0","findings_count":0},{"name":"ai_meta_artifact","ran_at":"2026-05-19T20:38:34.676581Z","status":"completed","version":"1.0.0","findings_count":0},{"name":"doi_title_agreement","ran_at":"2026-05-19T13:31:19.132491Z","status":"completed","version":"1.0.0","findings_count":0},{"name":"doi_compliance","ran_at":"2026-05-19T10:32:28.411426Z","status":"completed","version":"1.0.0","findings_count":0}],"snapshot_sha256":"05b7a8d978e4d8ff60d3a0d514cd49df30b2f90df3ec338dd10be8fdfd0ae702"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"168bddb9-722f-4e2d-8eb3-aeaae40c2f37"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-11T00:08:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IpHhDvzS2mNwshY0P7vFNpciTHkl0Q/Njzum/8VXrr8IQHvonqecDZiIGPKQ5lqdpW3RZME9m8QD6f818kBKDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T14:14:35.785501Z"},"content_sha256":"1426e0732374cff97f01090bf53fe4614ddfd5e148c3b4eaf715d39ed7b0edef","schema_version":"1.0","event_id":"sha256:1426e0732374cff97f01090bf53fe4614ddfd5e148c3b4eaf715d39ed7b0edef"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/bundle.json","state_url":"https://pith.science/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-11T14:14:35Z","links":{"resolver":"https://pith.science/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW","bundle":"https://pith.science/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/bundle.json","state":"https://pith.science/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Q6JYRBA52K2D2R3IBPLEI6NOCW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:Q6JYRBA52K2D2R3IBPLEI6NOCW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4b89152d2063d8e7594c94602a53c3c7b9a3d6b5ead2bc613d34e5dc4121b420","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-05-09T17:05:44Z","title_canon_sha256":"7a1170a886a839d58bc029648a6f567746c011c4ad1a7d8271d38087b9aa7b5e"},"schema_version":"1.0","source":{"id":"2605.09059","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.09059","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"arxiv_version","alias_value":"2605.09059v2","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.09059","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_12","alias_value":"Q6JYRBA52K2D","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_16","alias_value":"Q6JYRBA52K2D2R3I","created_at":"2026-06-11T00:08:12Z"},{"alias_kind":"pith_short_8","alias_value":"Q6JYRBA5","created_at":"2026-06-11T00:08:12Z"}],"graph_snapshots":[{"event_id":"sha256:1426e0732374cff97f01090bf53fe4614ddfd5e148c3b4eaf715d39ed7b0edef","target":"graph","created_at":"2026-06-11T00:08:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"reviews gathered from developers can yield many new findings, especially those related to the code being in a production-ready state, that would not be possible to obtain using the standard correctness-focused benchmark approach."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That a structured code-review process with (unspecified number of) developers produces generalizable, unbiased insights into production readiness that standard benchmarks miss."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"A custom three-fold methodology combining a complex-project correctness benchmark, code quality verification, and structured developer reviews to evaluate LLM-generated code beyond correctness alone."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook."}],"snapshot_sha256":"4f1c40dc50666d95f215cfc0d18734115166fcba4cb135eeefa53f28bf833bcc"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[{"findings_count":0,"name":"claim_evidence","ran_at":"2026-05-20T08:22:01.706986Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"ai_meta_artifact","ran_at":"2026-05-19T20:38:34.676581Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_title_agreement","ran_at":"2026-05-19T13:31:19.132491Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_compliance","ran_at":"2026-05-19T10:32:28.411426Z","status":"completed","version":"1.0.0"}],"endpoint":"/pith/2605.09059/integrity.json","findings":[],"snapshot_sha256":"05b7a8d978e4d8ff60d3a0d514cd49df30b2f90df3ec338dd10be8fdfd0ae702","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Code generation is one of the tasks for which the use of Large Language Models is widely adopted and highly successful. Given this popularity, there are many benchmarks dedicated to code generation that can help select the best model. However, they primarily focus on measuring solution correctness, leaving other aspects, such as code quality and usability, behind. This paper aims to describe a custom tree-fold evaluation methodology for code generated by Large Language Models that bridges this gap. The methodology includes a dedicated correctness benchmark based on a complex multi-level comput","authors_text":"Anne Schwerk, Joanna Szych","cross_cats":[],"headline":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-05-09T17:05:44Z","title":"Evaluating LLM-Generated Code: A Benchmark and Developer Study"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2605.09059","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-12T02:22:34.681743Z","id":"168bddb9-722f-4e2d-8eb3-aeaae40c2f37","model_set":{"reader":"grok-4.3"},"one_line_summary":"A custom three-fold methodology combining a complex-project correctness benchmark, code quality verification, and structured developer reviews to evaluate LLM-generated code beyond correctness alone.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Developer reviews uncover production-readiness issues in LLM code that standard correctness benchmarks overlook.","strongest_claim":"reviews gathered from developers can yield many new findings, especially those related to the code being in a production-ready state, that would not be possible to obtain using the standard correctness-focused benchmark approach.","weakest_assumption":"That a structured code-review process with (unspecified number of) developers produces generalizable, unbiased insights into production readiness that standard benchmarks miss."}},"verdict_id":"168bddb9-722f-4e2d-8eb3-aeaae40c2f37"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:aa04f004931682ff397777a05e83d5df579cd3120a0710340383c13157128559","target":"record","created_at":"2026-06-11T00:08:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4b89152d2063d8e7594c94602a53c3c7b9a3d6b5ead2bc613d34e5dc4121b420","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-05-09T17:05:44Z","title_canon_sha256":"7a1170a886a839d58bc029648a6f567746c011c4ad1a7d8271d38087b9aa7b5e"},"schema_version":"1.0","source":{"id":"2605.09059","kind":"arxiv","version":2}},"canonical_sha256":"879388841dd2b43d47680bd64479ae15b5d9d59df3640a661e05d4f6c921f58c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"879388841dd2b43d47680bd64479ae15b5d9d59df3640a661e05d4f6c921f58c","first_computed_at":"2026-06-11T00:08:12.678488Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-06-11T00:08:12.678488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6gmTfvkoxaWF6dPK4WozeHjo7Df45rzw9F2tdsSq43jmDcujsUwBCn3y/DtPVtn2yOPgDDFO1czNpwSZHso1Cg==","signature_status":"signed_v1","signed_at":"2026-06-11T00:08:12.679456Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.09059","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:aa04f004931682ff397777a05e83d5df579cd3120a0710340383c13157128559","sha256:1426e0732374cff97f01090bf53fe4614ddfd5e148c3b4eaf715d39ed7b0edef"],"state_sha256":"15ae2cc038cd3a19fa42adae96d3b139451f48b51e320da7266b2d8ce86f16fd"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9zpPLj5rUmbvwAKMLtWKcJ8jfXgAls3BKCAOXzWvnvEqfvv8qC80kgp/2Y0pL+4zECoG9HdgEjHw5MSFzn3RDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-11T14:14:35.788395Z","bundle_sha256":"ddf0814e226b812dc70e407ec16504f5e494ad5d256cc53814d6b0be3f044c53"}}