{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:N4QUZRMPAXON7EY2BWPHNIHSZ2","short_pith_number":"pith:N4QUZRMP","canonical_record":{"source":{"id":"2211.09374","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"fc4183052d88ca415e365d6e4414a772f37fbbaa60f99d10770a33aa4ae9f6b8","abstract_canon_sha256":"fbe1964b247f1e186964a0a156bd8d7659c1da093afcdc0acff40b9a10937893"},"schema_version":"1.0"},"canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","source":{"kind":"arxiv","id":"2211.09374","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.09374","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"arxiv_version","alias_value":"2211.09374v1","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.09374","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_12","alias_value":"N4QUZRMPAXON","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_16","alias_value":"N4QUZRMPAXON7EY2","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_8","alias_value":"N4QUZRMP","created_at":"2026-07-05T05:16:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:N4QUZRMPAXON7EY2BWPHNIHSZ2","target":"record","payload":{"canonical_record":{"source":{"id":"2211.09374","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"fc4183052d88ca415e365d6e4414a772f37fbbaa60f99d10770a33aa4ae9f6b8","abstract_canon_sha256":"fbe1964b247f1e186964a0a156bd8d7659c1da093afcdc0acff40b9a10937893"},"schema_version":"1.0"},"canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:16:55.138591Z","signature_b64":"FhCnLQdnwJHZ/5kuS+owXD7vaYB3dUCL1avpmNQcuaPncStP4XoqpIzY0s9Q+lwfKr/kbQQUkgXcpZXNA3/6BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","last_reissued_at":"2026-07-05T05:16:55.138240Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:16:55.138240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2211.09374","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:16:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bmt35Q/ux2wM1lFZ7zbtdfqprij6ASZesIDRbamY2aGkDYuHv1V3kRzPlrurb8m6NJYLzgx6CxK9bX9MAkoBCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T16:46:49.982860Z"},"content_sha256":"b0643deb20beeba5f18465441ab5ce394d975842b50ae0b588255bdc3fd5ba9d","schema_version":"1.0","event_id":"sha256:b0643deb20beeba5f18465441ab5ce394d975842b50ae0b588255bdc3fd5ba9d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:N4QUZRMPAXON7EY2BWPHNIHSZ2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Execution-based Evaluation for Data Science Code Generation Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.SE","authors_text":"Chenglong Wang, Colin Clement, Cong Yan, Haotian Cui, Jeevana Priya Inala, Jianfeng Gao, Jipeng Zhang, Junjie Huang, Nan Duan","submitted_at":"2022-11-17T07:04:11Z","abstract_excerpt":"Code generation models can benefit data scientists' productivity by automatically generating code from context and text descriptions. An important measure of the modeling progress is whether a model can generate code that can correctly execute to solve the task. However, due to the lack of an evaluation dataset that directly supports execution-based model evaluation, existing work relies on code surface form similarity metrics (e.g., BLEU, CodeBLEU) for model selection, which can be inaccurate.\n  To remedy this, we introduce ExeDS, an evaluation dataset for execution evaluation for data scienc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.09374","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.09374/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:16:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uEQyJ/sMjRbk5qPZymBkbp17O/1roW7rN7rdNv0Uch11SH4usJ+D9obfBHBSbzcXTyGcrkgefcUvc79iqg2oDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T16:46:49.983568Z"},"content_sha256":"bf62803ff420f8bb22959b72d3fed90292377e73ab00e8837132c24d63c533a8","schema_version":"1.0","event_id":"sha256:bf62803ff420f8bb22959b72d3fed90292377e73ab00e8837132c24d63c533a8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/bundle.json","state_url":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T16:46:49Z","links":{"resolver":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2","bundle":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/bundle.json","state":"https://pith.science/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/N4QUZRMPAXON7EY2BWPHNIHSZ2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:N4QUZRMPAXON7EY2BWPHNIHSZ2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fbe1964b247f1e186964a0a156bd8d7659c1da093afcdc0acff40b9a10937893","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","title_canon_sha256":"fc4183052d88ca415e365d6e4414a772f37fbbaa60f99d10770a33aa4ae9f6b8"},"schema_version":"1.0","source":{"id":"2211.09374","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.09374","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"arxiv_version","alias_value":"2211.09374v1","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.09374","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_12","alias_value":"N4QUZRMPAXON","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_16","alias_value":"N4QUZRMPAXON7EY2","created_at":"2026-07-05T05:16:55Z"},{"alias_kind":"pith_short_8","alias_value":"N4QUZRMP","created_at":"2026-07-05T05:16:55Z"}],"graph_snapshots":[{"event_id":"sha256:bf62803ff420f8bb22959b72d3fed90292377e73ab00e8837132c24d63c533a8","target":"graph","created_at":"2026-07-05T05:16:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2211.09374/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Code generation models can benefit data scientists' productivity by automatically generating code from context and text descriptions. An important measure of the modeling progress is whether a model can generate code that can correctly execute to solve the task. However, due to the lack of an evaluation dataset that directly supports execution-based model evaluation, existing work relies on code surface form similarity metrics (e.g., BLEU, CodeBLEU) for model selection, which can be inaccurate.\n  To remedy this, we introduce ExeDS, an evaluation dataset for execution evaluation for data scienc","authors_text":"Chenglong Wang, Colin Clement, Cong Yan, Haotian Cui, Jeevana Priya Inala, Jianfeng Gao, Jipeng Zhang, Junjie Huang, Nan Duan","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","title":"Execution-based Evaluation for Data Science Code Generation Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.09374","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b0643deb20beeba5f18465441ab5ce394d975842b50ae0b588255bdc3fd5ba9d","target":"record","created_at":"2026-07-05T05:16:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fbe1964b247f1e186964a0a156bd8d7659c1da093afcdc0acff40b9a10937893","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2022-11-17T07:04:11Z","title_canon_sha256":"fc4183052d88ca415e365d6e4414a772f37fbbaa60f99d10770a33aa4ae9f6b8"},"schema_version":"1.0","source":{"id":"2211.09374","kind":"arxiv","version":1}},"canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6f214cc58f05dcdf931a0d9e76a0f2ce998fbfc66f3fa2f5f124736796389b10","first_computed_at":"2026-07-05T05:16:55.138240Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:16:55.138240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FhCnLQdnwJHZ/5kuS+owXD7vaYB3dUCL1avpmNQcuaPncStP4XoqpIzY0s9Q+lwfKr/kbQQUkgXcpZXNA3/6BQ==","signature_status":"signed_v1","signed_at":"2026-07-05T05:16:55.138591Z","signed_message":"canonical_sha256_bytes"},"source_id":"2211.09374","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b0643deb20beeba5f18465441ab5ce394d975842b50ae0b588255bdc3fd5ba9d","sha256:bf62803ff420f8bb22959b72d3fed90292377e73ab00e8837132c24d63c533a8"],"state_sha256":"a05655152fe618d4bda1427db6e83cd621427ad41fefe4a5e4de2593b8bccfa4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cCUhT5qJEifbT3zmLnB+O2ONY8dC45G0xXwHpF3G831kQRKxoWvI5GE0hWoZurc+vwyCIJ0i1V1DE3w+eHkUBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T16:46:49.989293Z","bundle_sha256":"4ddc2124e31bee7565eb40c7fa11eac396ec8daea8fe4d4034de825b32c3d09e"}}