{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:D4JTGDMBJDPA7DI3JDFP6UOJQM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263"},"schema_version":"1.0","source":{"id":"2409.19014","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"arxiv_version","alias_value":"2409.19014v4","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_12","alias_value":"D4JTGDMBJDPA","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_16","alias_value":"D4JTGDMBJDPA7DI3","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_8","alias_value":"D4JTGDMB","created_at":"2026-07-05T09:27:05Z"}],"graph_snapshots":[{"event_id":"sha256:48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa","target":"graph","created_at":"2026-07-05T09:27:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.19014/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Text-to-SQL systems have become crucial for translating natural language into SQL queries in various industries, enabling non-technical users to perform complex data operations. The need for accurate evaluation methods has increased as these systems have grown more sophisticated. However, the Execution Accuracy (EX), the most prevalent evaluation metric, still shows many false positives and negatives. Thus, this paper introduces FLEX (False-Less EXecution), a novel approach to evaluating text-to-SQL systems using large language models (LLMs) to emulate human expert-level evaluation of SQL quer","authors_text":"Heegyu Kim, Hyunsouk Cho, Seunghwan Choi, Seungtaek Choi, Taeyang Jeon","cross_cats":["cs.IR","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title":"FLEX: Expert-level False-Less EXecution Metric for Reliable Text-to-SQL Benchmark"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.19014","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365","target":"record","created_at":"2026-07-05T09:27:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263"},"schema_version":"1.0","source":{"id":"2409.19014","kind":"arxiv","version":4}},"canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","first_computed_at":"2026-07-05T09:27:05.276239Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:27:05.276239Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8XifsVq/fFuBoVRDxFp4s9QIzfZ1PX7afNCORl8j0Ty/XjOuk1+oakl8/QgLLQruwx9L5Zv2m9kHmFyuoCMIDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:27:05.276720Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.19014","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365","sha256:48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa"],"state_sha256":"3c0f2298be57546de054d45a1ef9a18c437cf9b35d106a3b2dabcf4e5f1813c0"}