{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:D4JTGDMBJDPA7DI3JDFP6UOJQM","short_pith_number":"pith:D4JTGDMB","canonical_record":{"source":{"id":"2409.19014","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","cross_cats_sorted":["cs.IR","cs.LG"],"title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263","abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f"},"schema_version":"1.0"},"canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","source":{"kind":"arxiv","id":"2409.19014","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"arxiv_version","alias_value":"2409.19014v4","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_12","alias_value":"D4JTGDMBJDPA","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_16","alias_value":"D4JTGDMBJDPA7DI3","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_8","alias_value":"D4JTGDMB","created_at":"2026-07-05T09:27:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:D4JTGDMBJDPA7DI3JDFP6UOJQM","target":"record","payload":{"canonical_record":{"source":{"id":"2409.19014","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","cross_cats_sorted":["cs.IR","cs.LG"],"title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263","abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f"},"schema_version":"1.0"},"canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:27:05.276720Z","signature_b64":"8XifsVq/fFuBoVRDxFp4s9QIzfZ1PX7afNCORl8j0Ty/XjOuk1+oakl8/QgLLQruwx9L5Zv2m9kHmFyuoCMIDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","last_reissued_at":"2026-07-05T09:27:05.276239Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:27:05.276239Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.19014","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:27:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mPBstaP0W+iO27S/4PT68xxdMya04bGvjjY9JcvZH1nZd8oDzEEUS43iAnIURCtP1fCDMYRZLcqKq5xf64bNCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T15:26:51.976264Z"},"content_sha256":"7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365","schema_version":"1.0","event_id":"sha256:7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:D4JTGDMBJDPA7DI3JDFP6UOJQM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"FLEX: Expert-level False-Less EXecution Metric for Reliable Text-to-SQL Benchmark","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.IR","cs.LG"],"primary_cat":"cs.CL","authors_text":"Heegyu Kim, Hyunsouk Cho, Seunghwan Choi, Seungtaek Choi, Taeyang Jeon","submitted_at":"2024-09-24T01:40:50Z","abstract_excerpt":"Text-to-SQL systems have become crucial for translating natural language into SQL queries in various industries, enabling non-technical users to perform complex data operations. The need for accurate evaluation methods has increased as these systems have grown more sophisticated. However, the Execution Accuracy (EX), the most prevalent evaluation metric, still shows many false positives and negatives. Thus, this paper introduces FLEX (False-Less EXecution), a novel approach to evaluating text-to-SQL systems using large language models (LLMs) to emulate human expert-level evaluation of SQL quer"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.19014","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.19014/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:27:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vpkeWXlX0z17vFDy6w397G0buxw/y7Jc3l7jJWyj+oV50DaSm9blhScJ7yWGTXwBXKU2pQTettEqIUAs+v5RAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T15:26:51.976746Z"},"content_sha256":"48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa","schema_version":"1.0","event_id":"sha256:48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/bundle.json","state_url":"https://pith.science/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T15:26:51Z","links":{"resolver":"https://pith.science/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM","bundle":"https://pith.science/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/bundle.json","state":"https://pith.science/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/D4JTGDMBJDPA7DI3JDFP6UOJQM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:D4JTGDMBJDPA7DI3JDFP6UOJQM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263"},"schema_version":"1.0","source":{"id":"2409.19014","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"arxiv_version","alias_value":"2409.19014v4","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.19014","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_12","alias_value":"D4JTGDMBJDPA","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_16","alias_value":"D4JTGDMBJDPA7DI3","created_at":"2026-07-05T09:27:05Z"},{"alias_kind":"pith_short_8","alias_value":"D4JTGDMB","created_at":"2026-07-05T09:27:05Z"}],"graph_snapshots":[{"event_id":"sha256:48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa","target":"graph","created_at":"2026-07-05T09:27:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.19014/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Text-to-SQL systems have become crucial for translating natural language into SQL queries in various industries, enabling non-technical users to perform complex data operations. The need for accurate evaluation methods has increased as these systems have grown more sophisticated. However, the Execution Accuracy (EX), the most prevalent evaluation metric, still shows many false positives and negatives. Thus, this paper introduces FLEX (False-Less EXecution), a novel approach to evaluating text-to-SQL systems using large language models (LLMs) to emulate human expert-level evaluation of SQL quer","authors_text":"Heegyu Kim, Hyunsouk Cho, Seunghwan Choi, Seungtaek Choi, Taeyang Jeon","cross_cats":["cs.IR","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title":"FLEX: Expert-level False-Less EXecution Metric for Reliable Text-to-SQL Benchmark"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.19014","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365","target":"record","created_at":"2026-07-05T09:27:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"411a68ae7d8d1ff2e14169e8c028ac7769c6915886255418ef962f48d9b0411f","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T01:40:50Z","title_canon_sha256":"fc3ba7e03276df41456e6289395ce3676e6ee0677a973d34de508de4277c6263"},"schema_version":"1.0","source":{"id":"2409.19014","kind":"arxiv","version":4}},"canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1f13330d8148de0f8d1b48caff51c98314682b9b4e84d428d23a7950ab83d3aa","first_computed_at":"2026-07-05T09:27:05.276239Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:27:05.276239Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8XifsVq/fFuBoVRDxFp4s9QIzfZ1PX7afNCORl8j0Ty/XjOuk1+oakl8/QgLLQruwx9L5Zv2m9kHmFyuoCMIDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:27:05.276720Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.19014","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7c9d4cad495473a917ebc4787e631e834bbc463c05192122e087b4eac393b365","sha256:48a8848176a7cf22954cd5ad20181be2abccef2becd5ef6ec3a14f1d77f136fa"],"state_sha256":"3c0f2298be57546de054d45a1ef9a18c437cf9b35d106a3b2dabcf4e5f1813c0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uwK6TaGMoq78DSQDEezohjG7qVsV/OnnQofBWIICaCxB9tdhqAA8GyzbcT/DC8x6CL43FqNhKxmmZq72p5nHCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T15:26:51.979877Z","bundle_sha256":"c4232f24b9fb3ef82dcf0aebbef5334e7b52565f6708ef1db9a80afb72e320d6"}}