{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:ALB2MYQKPB5R3RK6KXIWCWHRLL","short_pith_number":"pith:ALB2MYQK","canonical_record":{"source":{"id":"2405.01580","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c","abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36"},"schema_version":"1.0"},"canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","source":{"kind":"arxiv","id":"2405.01580","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"arxiv_version","alias_value":"2405.01580v1","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_12","alias_value":"ALB2MYQKPB5R","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_16","alias_value":"ALB2MYQKPB5R3RK6","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_8","alias_value":"ALB2MYQK","created_at":"2026-07-05T08:14:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:ALB2MYQKPB5R3RK6KXIWCWHRLL","target":"record","payload":{"canonical_record":{"source":{"id":"2405.01580","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c","abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36"},"schema_version":"1.0"},"canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:14:58.390304Z","signature_b64":"MkRbEJf+X/Ct3Tmml0vKAVteYSTbjeb9beIuXzTZlQ20JGniIypf62dvIfGXsmwT8ayvzia6KOfwHCLAfzetBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","last_reissued_at":"2026-07-05T08:14:58.389899Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:14:58.389899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.01580","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:14:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8UFNHlAjGJ+NZg57HADi1lGWE9KWGN/pNzwQI+Btlo2eMnoccW/nGUsnKTqHu2nAp8B4UHe95sje82sBIbcgAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T00:18:27.861205Z"},"content_sha256":"4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7","schema_version":"1.0","event_id":"sha256:4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:ALB2MYQKPB5R3RK6KXIWCWHRLL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Limitations of Embedding Based Methods for Measuring Functional Correctness for Code Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.SE","authors_text":"Atharva Naik","submitted_at":"2024-04-26T15:54:39Z","abstract_excerpt":"The task of code generation from natural language (NL2Code) has become extremely popular, especially with the advent of Large Language Models (LLMs). However, efforts to quantify and track this progress have suffered due to a lack of reliable metrics for functional correctness. While popular benchmarks like HumanEval have test cases to enable reliable evaluation of correctness, it is time-consuming and requires human effort to collect test cases. As an alternative several reference-based evaluation metrics have been proposed, with embedding-based metrics like CodeBERTScore being touted as havi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01580","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.01580/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:14:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ux+VTj9sVpzXFJ2UY8jNpUtdRi/5IkxSh+70QZC/eKNAv5u2VvncOsRC3QS+YKYngGt0nyResuH0fYxy45CzBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T00:18:27.861686Z"},"content_sha256":"e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b","schema_version":"1.0","event_id":"sha256:e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/bundle.json","state_url":"https://pith.science/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T00:18:27Z","links":{"resolver":"https://pith.science/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL","bundle":"https://pith.science/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/bundle.json","state":"https://pith.science/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ALB2MYQKPB5R3RK6KXIWCWHRLL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ALB2MYQKPB5R3RK6KXIWCWHRLL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c"},"schema_version":"1.0","source":{"id":"2405.01580","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"arxiv_version","alias_value":"2405.01580v1","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01580","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_12","alias_value":"ALB2MYQKPB5R","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_16","alias_value":"ALB2MYQKPB5R3RK6","created_at":"2026-07-05T08:14:58Z"},{"alias_kind":"pith_short_8","alias_value":"ALB2MYQK","created_at":"2026-07-05T08:14:58Z"}],"graph_snapshots":[{"event_id":"sha256:e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b","target":"graph","created_at":"2026-07-05T08:14:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.01580/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The task of code generation from natural language (NL2Code) has become extremely popular, especially with the advent of Large Language Models (LLMs). However, efforts to quantify and track this progress have suffered due to a lack of reliable metrics for functional correctness. While popular benchmarks like HumanEval have test cases to enable reliable evaluation of correctness, it is time-consuming and requires human effort to collect test cases. As an alternative several reference-based evaluation metrics have been proposed, with embedding-based metrics like CodeBERTScore being touted as havi","authors_text":"Atharva Naik","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title":"On the Limitations of Embedding Based Methods for Measuring Functional Correctness for Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01580","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7","target":"record","created_at":"2026-07-05T08:14:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"61748d2dbca0a9377342d9de7c3db55c81c43dd0b4a6a7821b4d4bbb1404cf36","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-04-26T15:54:39Z","title_canon_sha256":"01dfb9ac121d0e7777adccaae6b631d03709ea49fd71953bb0bf4fb82205e92c"},"schema_version":"1.0","source":{"id":"2405.01580","kind":"arxiv","version":1}},"canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"02c3a6620a787b1dc55e55d16158f15add27ba2c1be45604de98d7a385b6b764","first_computed_at":"2026-07-05T08:14:58.389899Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:14:58.389899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MkRbEJf+X/Ct3Tmml0vKAVteYSTbjeb9beIuXzTZlQ20JGniIypf62dvIfGXsmwT8ayvzia6KOfwHCLAfzetBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:14:58.390304Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.01580","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4e943e37fd72113cbf571b7ac0365d0d55cbaa88415a1af95d82958cb7b2ffa7","sha256:e1e7c62a219bea10abb5869411c8a7c07840884e92b351c4c66aa6926b00ff7b"],"state_sha256":"d5746709e391acd66ad634ff4dfd819a5bf16972dcf673519bf44d7beca2909b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Uo1e8mlLiqszllc8BAXJKQMBNZvvyKRSY2ajeRQfY4glFDxAzCCEVf/VEeIc6huN/nlXR9eVl9goXS68QrWNDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T00:18:27.866307Z","bundle_sha256":"e4f210e010c3ca32d17d366f26f4f282dae2052bec876fe075284bbdb7437507"}}