{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:GXZWOMWXDECNUCQRZHDBCWFDH3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6cbba20173a88250d99dfd19ea8a1161529a5098f4f2da605e38a56fecf7ceac","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SD","submitted_at":"2026-05-13T17:58:52Z","title_canon_sha256":"21a1fdb87bd4786de47265dd4dc46819ebcd0c4bfa52c5571ff63f262e2ca61c"},"schema_version":"1.0","source":{"id":"2605.13841","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.13841","created_at":"2026-05-18T02:44:09Z"},{"alias_kind":"arxiv_version","alias_value":"2605.13841v1","created_at":"2026-05-18T02:44:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.13841","created_at":"2026-05-18T02:44:09Z"},{"alias_kind":"pith_short_12","alias_value":"GXZWOMWXDECN","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"GXZWOMWXDECNUCQR","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"GXZWOMWX","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:fd91e56378ac5c80548b69929616872e8bac0948a57e3439779ecc7582c62136","target":"graph","created_at":"2026-05-18T02:44:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"no system simultaneously exceeds 0.5 on both EVA-A pass@1 and EVA-X pass@1; peak and reliable performance diverge substantially (median pass@k - pass^k gap of 0.44 on EVA-A); accent and noise perturbations expose substantial robustness gaps"},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That bot-to-bot simulated conversations with automatic validation sufficiently capture the distribution of real human voice interactions and that the composite metrics EVA-A and EVA-X align with downstream user satisfaction or task success in production."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"EVA-Bench introduces a simulation-plus-scoring framework for voice agents that reveals no tested system exceeds 0.5 on both accuracy and experience metrics at pass@1."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"No voice agent exceeds 0.5 on both accuracy and experience metrics simultaneously."}],"snapshot_sha256":"2c8fb69efeef6b81869f14771170f891a5ca30df9fc7a5e421b13f030acf853d"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"6f3b24ac3ecc183e157eae3c69482e8f7688ccb9849b28615fe77c25c586fc27"},"paper":{"abstract_excerpt":"Voice agents, artificial intelligence systems that conduct spoken conversations to complete tasks, are increasingly deployed across enterprise applications. However, no existing benchmark jointly addresses two core evaluation challenges: generating realistic simulated conversations, and measuring quality across the full scope of voice-specific failure modes. We present EVA-Bench, an end-to-end evaluation framework that addresses both. On the simulation side, EVA-Bench orchestrates bot-to-bot audio conversations over dynamic multi-turn dialogues, with automatic simulation validation that detect","authors_text":"Anil Madamala, Fanny Riols, Gabrielle Gauthier Melan\\c{c}on, Hari Subramani, Hoang H. Nguyen, Joseph Marinier, Katrina Stankiewicz, Lindsay Devon Brin, Oluwanifemi Bamgbose, Raghav Mehndiratta, Sridhar Krishna Nemala, Srinivas Sunkara, Tara Bogavelli","cross_cats":["cs.AI","cs.CL","cs.LG"],"headline":"No voice agent exceeds 0.5 on both accuracy and experience metrics simultaneously.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SD","submitted_at":"2026-05-13T17:58:52Z","title":"EVA-Bench: A New End-to-end Framework for Evaluating Voice Agents"},"references":{"count":134,"internal_anchors":5,"resolved_work":134,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Andres, Vadim Fedorov, Rida Sadek, Enric Spagnolo-Arrizabalaga, and Nadescha Trudel","work_id":"bba6d2c3-5523-446f-8d25-2115671034ac","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Sd-eval: A benchmark dataset for spoken dialogue understanding beyond words.Advances in Neural Information Processing Systems, 37:56898–56918, 2024","work_id":"572ff470-f770-46e7-aad7-29a2d32bd56d","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Talking turns: Bench- marking audio foundation models on turn-taking dynamics","work_id":"6d79e34d-eaaf-44ae-b6c3-d5b71a23cdce","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Beyond task completion: Revealing corrupt success in LLM agents through procedure-aware evaluation.arXiv preprint arXiv:2603.03116, 2026","work_id":"1827c2b2-960e-4554-825e-b26fb4f8ca26","year":2026},{"cited_arxiv_id":"2410.17196","doi":"","is_internal_anchor":true,"ref_index":5,"title":"VoiceBench: Benchmarking LLM-Based Voice Assistants","work_id":"2e4e260a-a952-42ae-9dd6-2de2d3127881","year":2024}],"snapshot_sha256":"d67196f498c3a458b3e4c6a4be3cc199ea69d8980d9af3146634042d6461dacc"},"source":{"id":"2605.13841","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-14T17:26:52.036391Z","id":"cb107c6c-44a4-4c82-9a24-abf875aeb3bd","model_set":{"reader":"grok-4.3"},"one_line_summary":"EVA-Bench introduces a simulation-plus-scoring framework for voice agents that reveals no tested system exceeds 0.5 on both accuracy and experience metrics at pass@1.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"No voice agent exceeds 0.5 on both accuracy and experience metrics simultaneously.","strongest_claim":"no system simultaneously exceeds 0.5 on both EVA-A pass@1 and EVA-X pass@1; peak and reliable performance diverge substantially (median pass@k - pass^k gap of 0.44 on EVA-A); accent and noise perturbations expose substantial robustness gaps","weakest_assumption":"That bot-to-bot simulated conversations with automatic validation sufficiently capture the distribution of real human voice interactions and that the composite metrics EVA-A and EVA-X align with downstream user satisfaction or task success in production."}},"verdict_id":"cb107c6c-44a4-4c82-9a24-abf875aeb3bd"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7b6dfd764587d124af6142a6ca4adcf555ee4d3957943b141ddd2d110b08be68","target":"record","created_at":"2026-05-18T02:44:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6cbba20173a88250d99dfd19ea8a1161529a5098f4f2da605e38a56fecf7ceac","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SD","submitted_at":"2026-05-13T17:58:52Z","title_canon_sha256":"21a1fdb87bd4786de47265dd4dc46819ebcd0c4bfa52c5571ff63f262e2ca61c"},"schema_version":"1.0","source":{"id":"2605.13841","kind":"arxiv","version":1}},"canonical_sha256":"35f36732d71904da0a11c9c61158a33ec49e949722e8faaddfe924035a59771b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"35f36732d71904da0a11c9c61158a33ec49e949722e8faaddfe924035a59771b","first_computed_at":"2026-05-18T02:44:09.529740Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T02:44:09.529740Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"IbG+W0QYYPPfO2XXpoDTz/tI9LwTnK2i8DD960cOvBhwI6Hzi87JJfzjc9mnyIW7UnS1GLahJTCNa/8djFQ0BQ==","signature_status":"signed_v1","signed_at":"2026-05-18T02:44:09.530349Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.13841","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7b6dfd764587d124af6142a6ca4adcf555ee4d3957943b141ddd2d110b08be68","sha256:fd91e56378ac5c80548b69929616872e8bac0948a57e3439779ecc7582c62136"],"state_sha256":"a2e1827b72eeb18c8f34c0553ad887f7ad37ce7e0dd2cbbf7cf217ad4e7a774c"}