{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:5SML6EMNU4NV2L5UM2OQPVRF2T","short_pith_number":"pith:5SML6EMN","canonical_record":{"source":{"id":"2501.18197","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-01-30T08:31:09Z","cross_cats_sorted":["cs.DB"],"title_canon_sha256":"10fe38bf6b4c62f14a16a6a8b749a6d2cc0a2f1851ef0adb247d43b495c9e915","abstract_canon_sha256":"15076a302d1efe90151cd3a4d354a2966e3922ae54a1de5d9236b017109e13d0"},"schema_version":"1.0"},"canonical_sha256":"ec98bf118da71b5d2fb4669d07d625d4c03a6798329a5896ff43381511dda438","source":{"kind":"arxiv","id":"2501.18197","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.18197","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"arxiv_version","alias_value":"2501.18197v1","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.18197","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_12","alias_value":"5SML6EMNU4NV","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_16","alias_value":"5SML6EMNU4NV2L5U","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_8","alias_value":"5SML6EMN","created_at":"2026-07-05T10:09:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:5SML6EMNU4NV2L5UM2OQPVRF2T","target":"record","payload":{"canonical_record":{"source":{"id":"2501.18197","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-01-30T08:31:09Z","cross_cats_sorted":["cs.DB"],"title_canon_sha256":"10fe38bf6b4c62f14a16a6a8b749a6d2cc0a2f1851ef0adb247d43b495c9e915","abstract_canon_sha256":"15076a302d1efe90151cd3a4d354a2966e3922ae54a1de5d9236b017109e13d0"},"schema_version":"1.0"},"canonical_sha256":"ec98bf118da71b5d2fb4669d07d625d4c03a6798329a5896ff43381511dda438","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:09:15.783881Z","signature_b64":"GGEN5yyU+ZxLwazuegSfdxW7MVmGQR0BMmDU/TTmG8rvmFnlOuBbQD/2+zWOxR4IEofoSPNK3H5nBz2raZMcCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ec98bf118da71b5d2fb4669d07d625d4c03a6798329a5896ff43381511dda438","last_reissued_at":"2026-07-05T10:09:15.783423Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:09:15.783423Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2501.18197","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5lh6bXaEmt3Pnuh8/eOSD3ef9Vmg8Pbc4KSI4CJltLOojRZyxPgyr4Q8e5aZToPZ11OjKfwCvc93N910zYOEAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:38:47.414713Z"},"content_sha256":"546306bedf1a98e6da7b20de796fc3471d337ee24c9b0a3756a684b813518ebf","schema_version":"1.0","event_id":"sha256:546306bedf1a98e6da7b20de796fc3471d337ee24c9b0a3756a684b813518ebf"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:5SML6EMNU4NV2L5UM2OQPVRF2T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Fundamental Challenges in Evaluating Text2SQL Solutions and Detecting Their Limitations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.DB"],"primary_cat":"cs.LG","authors_text":"Cedric Renggli, Ihab F. Ilyas, Theodoros Rekatsinas","submitted_at":"2025-01-30T08:31:09Z","abstract_excerpt":"In this work, we dive into the fundamental challenges of evaluating Text2SQL solutions and highlight potential failure causes and the potential risks of relying on aggregate metrics in existing benchmarks. We identify two largely unaddressed limitations in current open benchmarks: (1) data quality issues in the evaluation data, mainly attributed to the lack of capturing the probabilistic nature of translating a natural language description into a structured query (e.g., NL ambiguity), and (2) the bias introduced by using different match functions as approximations for SQL equivalence.\n  To put"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.18197","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.18197/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jTP5rnD6GBYtXkB84v6CDoOs2l+hchDUutXvJFE+PGlgNC2xPmqvI9I5/ERQAErTao5mVzZRy85SxKVFYRcyDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:38:47.415295Z"},"content_sha256":"47b980d071583b19d6700859cedcbfbf49ad9fa93da89adb028b6858d6a75a22","schema_version":"1.0","event_id":"sha256:47b980d071583b19d6700859cedcbfbf49ad9fa93da89adb028b6858d6a75a22"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/bundle.json","state_url":"https://pith.science/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T09:38:47Z","links":{"resolver":"https://pith.science/pith/5SML6EMNU4NV2L5UM2OQPVRF2T","bundle":"https://pith.science/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/bundle.json","state":"https://pith.science/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5SML6EMNU4NV2L5UM2OQPVRF2T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:5SML6EMNU4NV2L5UM2OQPVRF2T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"15076a302d1efe90151cd3a4d354a2966e3922ae54a1de5d9236b017109e13d0","cross_cats_sorted":["cs.DB"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-01-30T08:31:09Z","title_canon_sha256":"10fe38bf6b4c62f14a16a6a8b749a6d2cc0a2f1851ef0adb247d43b495c9e915"},"schema_version":"1.0","source":{"id":"2501.18197","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.18197","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"arxiv_version","alias_value":"2501.18197v1","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.18197","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_12","alias_value":"5SML6EMNU4NV","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_16","alias_value":"5SML6EMNU4NV2L5U","created_at":"2026-07-05T10:09:15Z"},{"alias_kind":"pith_short_8","alias_value":"5SML6EMN","created_at":"2026-07-05T10:09:15Z"}],"graph_snapshots":[{"event_id":"sha256:47b980d071583b19d6700859cedcbfbf49ad9fa93da89adb028b6858d6a75a22","target":"graph","created_at":"2026-07-05T10:09:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.18197/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this work, we dive into the fundamental challenges of evaluating Text2SQL solutions and highlight potential failure causes and the potential risks of relying on aggregate metrics in existing benchmarks. We identify two largely unaddressed limitations in current open benchmarks: (1) data quality issues in the evaluation data, mainly attributed to the lack of capturing the probabilistic nature of translating a natural language description into a structured query (e.g., NL ambiguity), and (2) the bias introduced by using different match functions as approximations for SQL equivalence.\n  To put","authors_text":"Cedric Renggli, Ihab F. Ilyas, Theodoros Rekatsinas","cross_cats":["cs.DB"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-01-30T08:31:09Z","title":"Fundamental Challenges in Evaluating Text2SQL Solutions and Detecting Their Limitations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.18197","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:546306bedf1a98e6da7b20de796fc3471d337ee24c9b0a3756a684b813518ebf","target":"record","created_at":"2026-07-05T10:09:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"15076a302d1efe90151cd3a4d354a2966e3922ae54a1de5d9236b017109e13d0","cross_cats_sorted":["cs.DB"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-01-30T08:31:09Z","title_canon_sha256":"10fe38bf6b4c62f14a16a6a8b749a6d2cc0a2f1851ef0adb247d43b495c9e915"},"schema_version":"1.0","source":{"id":"2501.18197","kind":"arxiv","version":1}},"canonical_sha256":"ec98bf118da71b5d2fb4669d07d625d4c03a6798329a5896ff43381511dda438","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ec98bf118da71b5d2fb4669d07d625d4c03a6798329a5896ff43381511dda438","first_computed_at":"2026-07-05T10:09:15.783423Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:09:15.783423Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GGEN5yyU+ZxLwazuegSfdxW7MVmGQR0BMmDU/TTmG8rvmFnlOuBbQD/2+zWOxR4IEofoSPNK3H5nBz2raZMcCw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:09:15.783881Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.18197","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:546306bedf1a98e6da7b20de796fc3471d337ee24c9b0a3756a684b813518ebf","sha256:47b980d071583b19d6700859cedcbfbf49ad9fa93da89adb028b6858d6a75a22"],"state_sha256":"b15bcdd3d712a643320a7e147c029656d5fea0f868c8bacbe809684403e424e3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"l4rVv29dbekXg3fq1VTuVDGPlcCsAzFsfs4wbagarZeL2rwH2pdXVOQWhKYxnub7UhnLfB3UFOUrxVGbpBcmBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T09:38:47.420598Z","bundle_sha256":"c73a656b5cb13b12b19dae708a9ce3296514fa2bd2e5f5830f8800ee8d6ac46c"}}