{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:UCH5PWOF7QIHGFDOPYMVRW2FTO","short_pith_number":"pith:UCH5PWOF","canonical_record":{"source":{"id":"2312.13010","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-20T13:22:41Z","cross_cats_sorted":[],"title_canon_sha256":"5a6bba019c25f78396c3860172ae82837bc1075766dc395859feffa808ad737f","abstract_canon_sha256":"f82e5695b3ec0d0a1c717f6d512cde914ec019994f33804d782a92a621d4e1c0"},"schema_version":"1.0"},"canonical_sha256":"a08fd7d9c5fc1073146e7e1958db459bb4659d19bbbd7e45013838433479baa2","source":{"kind":"arxiv","id":"2312.13010","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.13010","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"arxiv_version","alias_value":"2312.13010v3","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.13010","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"pith_short_12","alias_value":"UCH5PWOF7QIH","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"UCH5PWOF7QIHGFDO","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"UCH5PWOF","created_at":"2026-05-18T12:33:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:UCH5PWOF7QIHGFDOPYMVRW2FTO","target":"record","payload":{"canonical_record":{"source":{"id":"2312.13010","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-20T13:22:41Z","cross_cats_sorted":[],"title_canon_sha256":"5a6bba019c25f78396c3860172ae82837bc1075766dc395859feffa808ad737f","abstract_canon_sha256":"f82e5695b3ec0d0a1c717f6d512cde914ec019994f33804d782a92a621d4e1c0"},"schema_version":"1.0"},"canonical_sha256":"a08fd7d9c5fc1073146e7e1958db459bb4659d19bbbd7e45013838433479baa2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:53.608635Z","signature_b64":"PQgk4bChuxIdfZBZx2aVGNTqQIfcnOJmMc/Jc7hOtP70YtynZS/xmtgP++EC1HelTzfUumh5F1ENiMGV4QCqBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a08fd7d9c5fc1073146e7e1958db459bb4659d19bbbd7e45013838433479baa2","last_reissued_at":"2026-05-17T23:38:53.608051Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:53.608051Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.13010","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cvQDCONcvgzq8Cj3/s7rO9xK/eNZSarpIcHm0O5y+nyx905hngMqcOBSlh5dcZWCrLnOaMJkAGZfJI9GbClmAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T01:27:24.426596Z"},"content_sha256":"6e95e5bc385b16d628a029ffa49661f10d92c046e6aa8a65cc228aae7b9fe0ff","schema_version":"1.0","event_id":"sha256:6e95e5bc385b16d628a029ffa49661f10d92c046e6aa8a65cc228aae7b9fe0ff"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:UCH5PWOF7QIHGFDOPYMVRW2FTO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models.","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dong Huang, Heming Cui, Jie M.Zhang, Michael Luck, Qingwen Bu, Yuhao Qing","submitted_at":"2023-12-20T13:22:41Z","abstract_excerpt":"The advancement of natural language processing (NLP) has been significantly boosted by the development of transformer-based large language models (LLMs). These models have revolutionized NLP tasks, particularly in code generation, aiding developers in creating software with enhanced efficiency. Despite their advancements, challenges in balancing code snippet generation with effective test case generation and execution persist. To address these issues, this paper introduces Multi-Agent Assistant Code Generation (AgentCoder), a novel solution comprising a multi-agent framework with specialized a"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"AgentCoder (GPT-4) achieves 96.3% and 91.8% pass@1 in HumanEval and MBPP datasets with an overall token overhead of 56.9K and 66.3K, while state-of-the-art obtains only 90.2% and 78.9% pass@1 with an overall token overhead of 138.2K and 206.5K.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That iterative feedback from the test executor agent reliably improves code quality without introducing new errors or causing the programmer agent to overfit to the generated tests.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"A three-agent loop of code generation, test creation, and execution feedback lifts pass@1 to 96.3% on HumanEval and 91.8% on MBPP for GPT-4 while using roughly half the tokens of prior state-of-the-art.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"d7d147f6cd672ca4dadde590e6d51339b9f2d41662ec74d9b2ae391cf9b2e2ac"},"source":{"id":"2312.13010","kind":"arxiv","version":3},"verdict":{"id":"c49bfe8b-e8db-44f8-a272-b9990c4cec5d","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-15T03:53:52.500420Z","strongest_claim":"AgentCoder (GPT-4) achieves 96.3% and 91.8% pass@1 in HumanEval and MBPP datasets with an overall token overhead of 56.9K and 66.3K, while state-of-the-art obtains only 90.2% and 78.9% pass@1 with an overall token overhead of 138.2K and 206.5K.","one_line_summary":"A three-agent loop of code generation, test creation, and execution feedback lifts pass@1 to 96.3% on HumanEval and 91.8% on MBPP for GPT-4 while using roughly half the tokens of prior state-of-the-art.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That iterative feedback from the test executor agent reliably improves code quality without introducing new errors or causing the programmer agent to overfit to the generated tests.","pith_extraction_headline":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models."},"references":{"count":54,"sample":[{"doi":"","year":2021,"title":"Unified pre-training for program understanding and generation.arXiv preprint","work_id":"92715367-2460-4833-ac02-f642b8a19c59","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2021,"title":"Program Synthesis with Large Language Models","work_id":"fd241a05-03b9-4de2-9588-9d77ce176125","ref_index":2,"cited_arxiv_id":"2108.07732","is_internal_anchor":true},{"doi":"","year":2005,"title":"Language Models are Few-Shot Learners","work_id":"214732c0-2edd-44a0-af9e-28184a2b8279","ref_index":4,"cited_arxiv_id":"2005.14165","is_internal_anchor":true},{"doi":"","year":2023,"title":"arXiv preprint arXiv:2305.17126 , year=","work_id":"1447b78e-0a79-4af6-8cd4-93220e680d2b","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2021,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","ref_index":7,"cited_arxiv_id":"2107.03374","is_internal_anchor":true}],"resolved_work":54,"snapshot_sha256":"48be6ac9592ab27d7ac76ca1d4bded7b7e6ad7415ddccbb28045f6c495e83149","internal_anchors":17},"formal_canon":{"evidence_count":3,"snapshot_sha256":"6a16cf67c9bc1bf6a7e196afe950083b1cef270d27c523ea83dba8ecafc59db4"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"c49bfe8b-e8db-44f8-a272-b9990c4cec5d"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xZ7WMuFyeOFJAlvv509dtjKBbMs3mwBoaOQxn7uuxDF+DruVnAqPOYfi7/3tpqvg7zcto0XHYsNiOlCG/Fk+BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T01:27:24.427769Z"},"content_sha256":"1522319ceea6c9591d940ea5358333fcd4e987d5af1b998ddeb18cc026f87939","schema_version":"1.0","event_id":"sha256:1522319ceea6c9591d940ea5358333fcd4e987d5af1b998ddeb18cc026f87939"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/bundle.json","state_url":"https://pith.science/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-01T01:27:24Z","links":{"resolver":"https://pith.science/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO","bundle":"https://pith.science/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/bundle.json","state":"https://pith.science/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UCH5PWOF7QIHGFDOPYMVRW2FTO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:UCH5PWOF7QIHGFDOPYMVRW2FTO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f82e5695b3ec0d0a1c717f6d512cde914ec019994f33804d782a92a621d4e1c0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-20T13:22:41Z","title_canon_sha256":"5a6bba019c25f78396c3860172ae82837bc1075766dc395859feffa808ad737f"},"schema_version":"1.0","source":{"id":"2312.13010","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.13010","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"arxiv_version","alias_value":"2312.13010v3","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.13010","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"pith_short_12","alias_value":"UCH5PWOF7QIH","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"UCH5PWOF7QIHGFDO","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"UCH5PWOF","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:1522319ceea6c9591d940ea5358333fcd4e987d5af1b998ddeb18cc026f87939","target":"graph","created_at":"2026-05-17T23:38:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"AgentCoder (GPT-4) achieves 96.3% and 91.8% pass@1 in HumanEval and MBPP datasets with an overall token overhead of 56.9K and 66.3K, while state-of-the-art obtains only 90.2% and 78.9% pass@1 with an overall token overhead of 138.2K and 206.5K."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That iterative feedback from the test executor agent reliably improves code quality without introducing new errors or causing the programmer agent to overfit to the generated tests."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"A three-agent loop of code generation, test creation, and execution feedback lifts pass@1 to 96.3% on HumanEval and 91.8% on MBPP for GPT-4 while using roughly half the tokens of prior state-of-the-art."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models."}],"snapshot_sha256":"d7d147f6cd672ca4dadde590e6d51339b9f2d41662ec74d9b2ae391cf9b2e2ac"},"formal_canon":{"evidence_count":3,"snapshot_sha256":"6a16cf67c9bc1bf6a7e196afe950083b1cef270d27c523ea83dba8ecafc59db4"},"paper":{"abstract_excerpt":"The advancement of natural language processing (NLP) has been significantly boosted by the development of transformer-based large language models (LLMs). These models have revolutionized NLP tasks, particularly in code generation, aiding developers in creating software with enhanced efficiency. Despite their advancements, challenges in balancing code snippet generation with effective test case generation and execution persist. To address these issues, this paper introduces Multi-Agent Assistant Code Generation (AgentCoder), a novel solution comprising a multi-agent framework with specialized a","authors_text":"Dong Huang, Heming Cui, Jie M.Zhang, Michael Luck, Qingwen Bu, Yuhao Qing","cross_cats":[],"headline":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-20T13:22:41Z","title":"AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation"},"references":{"count":54,"internal_anchors":17,"resolved_work":54,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Unified pre-training for program understanding and generation.arXiv preprint","work_id":"92715367-2460-4833-ac02-f642b8a19c59","year":2021},{"cited_arxiv_id":"2108.07732","doi":"","is_internal_anchor":true,"ref_index":2,"title":"Program Synthesis with Large Language Models","work_id":"fd241a05-03b9-4de2-9588-9d77ce176125","year":2021},{"cited_arxiv_id":"2005.14165","doi":"","is_internal_anchor":true,"ref_index":4,"title":"Language Models are Few-Shot Learners","work_id":"214732c0-2edd-44a0-af9e-28184a2b8279","year":2005},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"arXiv preprint arXiv:2305.17126 , year=","work_id":"1447b78e-0a79-4af6-8cd4-93220e680d2b","year":2023},{"cited_arxiv_id":"2107.03374","doi":"","is_internal_anchor":true,"ref_index":7,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","year":2021}],"snapshot_sha256":"48be6ac9592ab27d7ac76ca1d4bded7b7e6ad7415ddccbb28045f6c495e83149"},"source":{"id":"2312.13010","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-15T03:53:52.500420Z","id":"c49bfe8b-e8db-44f8-a272-b9990c4cec5d","model_set":{"reader":"grok-4.3"},"one_line_summary":"A three-agent loop of code generation, test creation, and execution feedback lifts pass@1 to 96.3% on HumanEval and 91.8% on MBPP for GPT-4 while using roughly half the tokens of prior state-of-the-art.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"A multi-agent system divides code generation among three agents to reach higher accuracy with lower token cost than single models.","strongest_claim":"AgentCoder (GPT-4) achieves 96.3% and 91.8% pass@1 in HumanEval and MBPP datasets with an overall token overhead of 56.9K and 66.3K, while state-of-the-art obtains only 90.2% and 78.9% pass@1 with an overall token overhead of 138.2K and 206.5K.","weakest_assumption":"That iterative feedback from the test executor agent reliably improves code quality without introducing new errors or causing the programmer agent to overfit to the generated tests."}},"verdict_id":"c49bfe8b-e8db-44f8-a272-b9990c4cec5d"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6e95e5bc385b16d628a029ffa49661f10d92c046e6aa8a65cc228aae7b9fe0ff","target":"record","created_at":"2026-05-17T23:38:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f82e5695b3ec0d0a1c717f6d512cde914ec019994f33804d782a92a621d4e1c0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-20T13:22:41Z","title_canon_sha256":"5a6bba019c25f78396c3860172ae82837bc1075766dc395859feffa808ad737f"},"schema_version":"1.0","source":{"id":"2312.13010","kind":"arxiv","version":3}},"canonical_sha256":"a08fd7d9c5fc1073146e7e1958db459bb4659d19bbbd7e45013838433479baa2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a08fd7d9c5fc1073146e7e1958db459bb4659d19bbbd7e45013838433479baa2","first_computed_at":"2026-05-17T23:38:53.608051Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:53.608051Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"PQgk4bChuxIdfZBZx2aVGNTqQIfcnOJmMc/Jc7hOtP70YtynZS/xmtgP++EC1HelTzfUumh5F1ENiMGV4QCqBw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:53.608635Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.13010","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6e95e5bc385b16d628a029ffa49661f10d92c046e6aa8a65cc228aae7b9fe0ff","sha256:1522319ceea6c9591d940ea5358333fcd4e987d5af1b998ddeb18cc026f87939"],"state_sha256":"3ebfaf1823dbfbf858a19a4a66a3d98564aa368d08ef85e99c7579da6ee7ccb9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Fz7R/OkoLhnSp+8viensbelmFnMNhSlCLr5Rq5EnJdnyg37nnHT5h3ySapoDSTevZlBNchnwCtAJVai5TbwyDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-01T01:27:24.432888Z","bundle_sha256":"5fa5ba45f12b7ffbbc586db495f81dd8f78ed128898b233ec817ab9eec2b45e8"}}