{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:FECJ2OC7FXNWG32QGNDT4M7SBA","short_pith_number":"pith:FECJ2OC7","schema_version":"1.0","canonical_sha256":"29049d385f2ddb636f5033473e33f20830f4da0d7027b1c516fa1d1c1d8062d4","source":{"kind":"arxiv","id":"2505.18223","version":2},"attestation_state":"computed","paper":{"title":"IDA-Bench: Evaluating LLMs on Interactive Guided Data Analysis","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hanyu Li, Haoyu Liu, Michael I. Jordan, Tianyu Guo, Tingyu Zhu, Xiaotie Deng, Zeyu Zheng","submitted_at":"2025-05-23T09:37:52Z","abstract_excerpt":"Large Language Models (LLMs) show promise as data analysis agents, but existing benchmarks overlook the iterative nature of the field, where experts' decisions evolve with deeper insights of the dataset. To address this, we introduce IDA-Bench, a novel benchmark evaluating LLM agents in multi-round interactive scenarios. Derived from complex Kaggle notebooks, tasks are presented as sequential natural language instructions by an LLM-simulated user. Agent performance is judged by comparing its final numerical output to the human-derived baseline. Initial results show that even state-of-the-art c"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.18223","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-23T09:37:52Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"2c7db140963df01fadf8a35a4539d075f324e4d3a1729077b2903ade4d3f5a83","abstract_canon_sha256":"03f2a8c70f4f50c8ae7d91052a2f684dd78427f09ed53b76b5d8e4f892c112c1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:16:58.448161Z","signature_b64":"yawGWob4Y6v1f+ATOE4Qy3vnyavUtqTBqA+D//XD5sGJlWNIFgeXiad5wZSffrg8SUFu6/bXrumBl+ei8XHLBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"29049d385f2ddb636f5033473e33f20830f4da0d7027b1c516fa1d1c1d8062d4","last_reissued_at":"2026-07-05T11:16:58.447596Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:16:58.447596Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"IDA-Bench: Evaluating LLMs on Interactive Guided Data Analysis","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hanyu Li, Haoyu Liu, Michael I. Jordan, Tianyu Guo, Tingyu Zhu, Xiaotie Deng, Zeyu Zheng","submitted_at":"2025-05-23T09:37:52Z","abstract_excerpt":"Large Language Models (LLMs) show promise as data analysis agents, but existing benchmarks overlook the iterative nature of the field, where experts' decisions evolve with deeper insights of the dataset. To address this, we introduce IDA-Bench, a novel benchmark evaluating LLM agents in multi-round interactive scenarios. Derived from complex Kaggle notebooks, tasks are presented as sequential natural language instructions by an LLM-simulated user. Agent performance is judged by comparing its final numerical output to the human-derived baseline. Initial results show that even state-of-the-art c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.18223","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.18223/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.18223","created_at":"2026-07-05T11:16:58.447657+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.18223v2","created_at":"2026-07-05T11:16:58.447657+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.18223","created_at":"2026-07-05T11:16:58.447657+00:00"},{"alias_kind":"pith_short_12","alias_value":"FECJ2OC7FXNW","created_at":"2026-07-05T11:16:58.447657+00:00"},{"alias_kind":"pith_short_16","alias_value":"FECJ2OC7FXNWG32Q","created_at":"2026-07-05T11:16:58.447657+00:00"},{"alias_kind":"pith_short_8","alias_value":"FECJ2OC7","created_at":"2026-07-05T11:16:58.447657+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.16000","citing_title":"GRACE-DS: a Guarded Reward-guided Agent Correction Environment in Data Science","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00051","citing_title":"Business Utility of Large Language Models as Exploratory Data Analysis Agents","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03808","citing_title":"Agentic-imodels: Evolving agentic interpretability tools via autoresearch","ref_index":34,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA","json":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA.json","graph_json":"https://pith.science/api/pith-number/FECJ2OC7FXNWG32QGNDT4M7SBA/graph.json","events_json":"https://pith.science/api/pith-number/FECJ2OC7FXNWG32QGNDT4M7SBA/events.json","paper":"https://pith.science/paper/FECJ2OC7"},"agent_actions":{"view_html":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA","download_json":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA.json","view_paper":"https://pith.science/paper/FECJ2OC7","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.18223&json=true","fetch_graph":"https://pith.science/api/pith-number/FECJ2OC7FXNWG32QGNDT4M7SBA/graph.json","fetch_events":"https://pith.science/api/pith-number/FECJ2OC7FXNWG32QGNDT4M7SBA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA/action/storage_attestation","attest_author":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA/action/author_attestation","sign_citation":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA/action/citation_signature","submit_replication":"https://pith.science/pith/FECJ2OC7FXNWG32QGNDT4M7SBA/action/replication_record"}},"created_at":"2026-07-05T11:16:58.447657+00:00","updated_at":"2026-07-05T11:16:58.447657+00:00"}