{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:N4QUZRMPAXON7EY2BWPHNIHSZ2","short_pith_number":"pith:N4QUZRMP","schema_version":"1.0","canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","source":{"kind":"arxiv","id":"2211.09374","version":1},"attestation_state":"computed","paper":{"title":"Execution-based Evaluation for Data Science Code Generation Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.SE","authors_text":"Chenglong Wang, Colin Clement, Cong Yan, Haotian Cui, Jeevana Priya Inala, Jianfeng Gao, Jipeng Zhang, Junjie Huang, Nan Duan","submitted_at":"2022-11-17T07:04:11Z","abstract_excerpt":"Code generation models can benefit data scientists' productivity by automatically generating code from context and text descriptions. An important measure of the modeling progress is whether a model can generate code that can correctly execute to solve the task. However, due to the lack of an evaluation dataset that directly supports execution-based model evaluation, existing work relies on code surface form similarity metrics (e.g., BLEU, CodeBLEU) for model selection, which can be inaccurate.\n  To remedy this, we introduce ExeDS, an evaluation dataset for execution evaluation for data scienc"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2211.09374","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"fc4183052d88ca415e365d6e4414a772f37fbbaa60f99d10770a33aa4ae9f6b8","abstract_canon_sha256":"fbe1964b247f1e186964a0a156bd8d7659c1da093afcdc0acff40b9a10937893"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:16:55.138591Z","signature_b64":"FhCnLQdnwJHZ/5kuS+owXD7vaYB3dUCL1avpmNQcuaPncStP4XoqpIzY0s9Q+lwfKr/kbQQUkgXcpZXNA3/6BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","last_reissued_at":"2026-07-05T05:16:55.138240Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:16:55.138240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Execution-based Evaluation for Data Science Code Generation Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.SE","authors_text":"Chenglong Wang, Colin Clement, Cong Yan, Haotian Cui, Jeevana Priya Inala, Jianfeng Gao, Jipeng Zhang, Junjie Huang, Nan Duan","submitted_at":"2022-11-17T07:04:11Z","abstract_excerpt":"Code generation models can benefit data scientists' productivity by automatically generating code from context and text descriptions. An important measure of the modeling progress is whether a model can generate code that can correctly execute to solve the task. However, due to the lack of an evaluation dataset that directly supports execution-based model evaluation, existing work relies on code surface form similarity metrics (e.g., BLEU, CodeBLEU) for model selection, which can be inaccurate.\n  To remedy this, we introduce ExeDS, an evaluation dataset for execution evaluation for data scienc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.09374","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.09374/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2211.09374","created_at":"2026-07-05T05:16:55.138295+00:00"},{"alias_kind":"arxiv_version","alias_value":"2211.09374v1","created_at":"2026-07-05T05:16:55.138295+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.09374","created_at":"2026-07-05T05:16:55.138295+00:00"},{"alias_kind":"pith_short_12","alias_value":"N4QUZRMPAXON","created_at":"2026-07-05T05:16:55.138295+00:00"},{"alias_kind":"pith_short_16","alias_value":"N4QUZRMPAXON7EY2","created_at":"2026-07-05T05:16:55.138295+00:00"},{"alias_kind":"pith_short_8","alias_value":"N4QUZRMP","created_at":"2026-07-05T05:16:55.138295+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2402.13228","citing_title":"Smaug: Fixing Failure Modes of Preference Optimisation with DPO-Positive","ref_index":127,"is_internal_anchor":false},{"citing_arxiv_id":"2406.00515","citing_title":"A Survey on Large Language Models for Code Generation","ref_index":109,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2","json":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2.json","graph_json":"https://pith.science/api/pith-number/N4QUZRMPAXON7EY2BWPHNIHSZ2/graph.json","events_json":"https://pith.science/api/pith-number/N4QUZRMPAXON7EY2BWPHNIHSZ2/events.json","paper":"https://pith.science/paper/N4QUZRMP"},"agent_actions":{"view_html":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2","download_json":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2.json","view_paper":"https://pith.science/paper/N4QUZRMP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2211.09374&json=true","fetch_graph":"https://pith.science/api/pith-number/N4QUZRMPAXON7EY2BWPHNIHSZ2/graph.json","fetch_events":"https://pith.science/api/pith-number/N4QUZRMPAXON7EY2BWPHNIHSZ2/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/action/timestamp_anchor","attest_storage":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/action/storage_attestation","attest_author":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/action/author_attestation","sign_citation":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/action/citation_signature","submit_replication":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/action/replication_record"}},"created_at":"2026-07-05T05:16:55.138295+00:00","updated_at":"2026-07-05T05:16:55.138295+00:00"}