{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:FAL4JNNFGADJ252EXPWX3ECZKQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"18964b1cece2710bd0d5489f437f4f7246a382cc6eab0a23b0b624827c6804ca","cross_cats_sorted":["cs.AI","cs.PL","cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-21T10:18:37Z","title_canon_sha256":"9f454b5fd5666f3fd6b9b1f77c86081aef821f4aef5eae5e56439466fc5afe5f"},"schema_version":"1.0","source":{"id":"2207.10397","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2207.10397","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"arxiv_version","alias_value":"2207.10397v2","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2207.10397","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"pith_short_12","alias_value":"FAL4JNNFGADJ","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"FAL4JNNFGADJ252E","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"FAL4JNNF","created_at":"2026-05-18T12:33:33Z"}],"graph_snapshots":[{"event_id":"sha256:94b3775d200202d58f2c43c5b1981c75ef45ffd95cc6a01627cbb7849e1046d5","target":"graph","created_at":"2026-05-17T23:38:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"CodeT improves the pass@1 metric on HumanEval to 65.8%, an absolute improvement of 18.8% over the code-davinci-002 model and more than 20% over previous state-of-the-art results."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That agreement between independently generated code samples on independently generated tests reliably indicates functional correctness rather than shared bugs or test weaknesses."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"CodeT improves code generation accuracy by using the same model to create test cases and then selecting solutions via output agreement on those tests, raising HumanEval pass@1 from 47% to 65.8%."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"CodeT generates test cases with the same model to select correct code samples via dual execution agreement."}],"snapshot_sha256":"c0b39ab2ff4903c5967a93fb2485db4b52184e302c306e58cded605d5fbb8151"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"The task of generating code solutions for a given programming problem can benefit from the use of pre-trained language models such as Codex, which can produce multiple diverse samples. However, a major challenge for this task is to select the most appropriate solution from the multiple samples generated by the pre-trained language models. A natural way to evaluate the quality and correctness of a code solution is to run it against a set of test cases, but the manual creation of such test cases is often costly and time-consuming. In this paper, we propose a novel method, CodeT, that leverages t","authors_text":"Anh Nguyen, Bei Chen, Daoguang Zan, Fengji Zhang, Jian-Guang Lou, Weizhu Chen, Zeqi Lin","cross_cats":["cs.AI","cs.PL","cs.SE"],"headline":"CodeT generates test cases with the same model to select correct code samples via dual execution agreement.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-21T10:18:37Z","title":"CodeT: Code Generation with Generated Tests"},"references":{"count":17,"internal_anchors":6,"resolved_work":17,"sample":[{"cited_arxiv_id":"2108.07732","doi":"","is_internal_anchor":true,"ref_index":1,"title":"Program Synthesis with Large Language Models","work_id":"fd241a05-03b9-4de2-9588-9d77ce176125","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Language models are few-shot learners","work_id":"f93ff324-f230-4a46-97b9-6b103c35585d","year":1901},{"cited_arxiv_id":"2107.03374","doi":"","is_internal_anchor":true,"ref_index":3,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","year":null},{"cited_arxiv_id":"2110.14168","doi":"","is_internal_anchor":true,"ref_index":4,"title":"Training Verifiers to Solve Math Word Problems","work_id":"acab1aa8-b4d6-40e0-a3ee-25341701dca2","year":null},{"cited_arxiv_id":"2204.05999","doi":"","is_internal_anchor":true,"ref_index":5,"title":"InCoder: A Generative Model for Code Infilling and Synthesis","work_id":"e98a5559-529d-48b1-833c-b85b662f190c","year":null}],"snapshot_sha256":"0ae46256bda4d7fa4217c296ab130a7ddb6b28ef879e79fcf162e163703d5246"},"source":{"id":"2207.10397","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-15T23:51:28.385332Z","id":"e21bdbb2-d0be-44ec-976c-a1954b6c04ea","model_set":{"reader":"grok-4.3"},"one_line_summary":"CodeT improves code generation accuracy by using the same model to create test cases and then selecting solutions via output agreement on those tests, raising HumanEval pass@1 from 47% to 65.8%.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"CodeT generates test cases with the same model to select correct code samples via dual execution agreement.","strongest_claim":"CodeT improves the pass@1 metric on HumanEval to 65.8%, an absolute improvement of 18.8% over the code-davinci-002 model and more than 20% over previous state-of-the-art results.","weakest_assumption":"That agreement between independently generated code samples on independently generated tests reliably indicates functional correctness rather than shared bugs or test weaknesses."}},"verdict_id":"e21bdbb2-d0be-44ec-976c-a1954b6c04ea"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:440a5ed432d0374e915460c37229a49c13dd9bb9121de1c79a71f1d5af5555d8","target":"record","created_at":"2026-05-17T23:38:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"18964b1cece2710bd0d5489f437f4f7246a382cc6eab0a23b0b624827c6804ca","cross_cats_sorted":["cs.AI","cs.PL","cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-21T10:18:37Z","title_canon_sha256":"9f454b5fd5666f3fd6b9b1f77c86081aef821f4aef5eae5e56439466fc5afe5f"},"schema_version":"1.0","source":{"id":"2207.10397","kind":"arxiv","version":2}},"canonical_sha256":"2817c4b5a530069d7744bbed7d905954374e394adc098726acb9656241cd5447","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2817c4b5a530069d7744bbed7d905954374e394adc098726acb9656241cd5447","first_computed_at":"2026-05-17T23:38:49.699265Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:49.699265Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NeEAjfKZoxQ3kYphp37Hfmn7BamgbFImalAkI5QblZoplH2SLQbJ+S9mfQkTFnETrodP/KPuNv21nYnLmNtkBg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:49.699762Z","signed_message":"canonical_sha256_bytes"},"source_id":"2207.10397","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:440a5ed432d0374e915460c37229a49c13dd9bb9121de1c79a71f1d5af5555d8","sha256:94b3775d200202d58f2c43c5b1981c75ef45ffd95cc6a01627cbb7849e1046d5"],"state_sha256":"5ee04963bbd819bd6bcf2e0200d5557d2039b45d9f7cab21b145acd255c62dcc"}