{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ALB2MYQKPB5R3RK6KXIWCWHRLL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c"},"schema_version":"1.0","source":{"id":"2405.01580","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"arxiv_version","alias_value":"2405.01580v1","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_12","alias_value":"ALB2MYQKPB5R","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_16","alias_value":"ALB2MYQKPB5R3RK6","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_8","alias_value":"ALB2MYQK","created_at":"2026-07-05T08:14:58Z"}],"graph_snapshots":[{"event_id":"sha256:e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b","target":"graph","created_at":"2026-07-05T08:14:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.01580/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The task of code generation from natural language (NL2Code) has become extremely popular, especially with the advent of Large Language Models (LLMs). However, efforts to quantify and track this progress have suffered due to a lack of reliable metrics for functional correctness. While popular benchmarks like HumanEval have test cases to enable reliable evaluation of correctness, it is time-consuming and requires human effort to collect test cases. As an alternative several reference-based evaluation metrics have been proposed, with embedding-based metrics like CodeBERTScore being touted as havi","authors_text":"Atharva Naik","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title":"On the Limitations of Embedding Based Methods for Measuring Functional Correctness for Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01580","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7","target":"record","created_at":"2026-07-05T08:14:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c"},"schema_version":"1.0","source":{"id":"2405.01580","kind":"arxiv","version":1}},"canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","first_computed_at":"2026-07-05T08:14:58.389899Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:14:58.389899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MkRbEJf+X/Ct3Tmml0vKAVteYSTbjeb9beIuXzTZlQ20JGniIypf62dvIfGXsmwT8ayvzia6KOfwHCLAfzetBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:14:58.390304Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.01580","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7","sha256:e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b"],"state_sha256":"d5746709e391acd66ad634ff4dfd819a5bf16972dcf673519bf44d7beca2909b"}