{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:HSF6KFQPKQVAIQQDFTPH7W4JIT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"521dba05ed6bf22d8571d21f417abcce37166426e5e95b0f100751c740bf4d2a","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2026-08-06T17:39:21Z","title_canon_sha256":"b6940799c67787c610d525435804dce867564d5f23638706170efc0acf3473e0"},"schema_version":"1.0","source":{"id":"2608.06329","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2608.06329","created_at":"2026-08-07T01:41:22Z"},{"alias_kind":"arxiv_version","alias_value":"2608.06329v1","created_at":"2026-08-07T01:41:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2608.06329","created_at":"2026-08-07T01:41:22Z"},{"alias_kind":"pith_short_12","alias_value":"HSF6KFQPKQVA","created_at":"2026-08-07T01:41:22Z"},{"alias_kind":"pith_short_16","alias_value":"HSF6KFQPKQVAIQQD","created_at":"2026-08-07T01:41:22Z"},{"alias_kind":"pith_short_8","alias_value":"HSF6KFQP","created_at":"2026-08-07T01:41:22Z"}],"graph_snapshots":[{"event_id":"sha256:214068e1903e2ae141203fe7fecb0953adadf7f50e90adb58d778fc8683b469c","target":"graph","created_at":"2026-08-07T01:41:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2608.06329/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Task-oriented conversational agents are evaluated using curated or automatically generated benchmarks, yet benchmark quality is rarely assessed. Poor benchmarks may contain inconsistent tasks, simplistic scenarios, or limited policy coverage, leading to unreliable evaluations. We introduce a reference-free framework that uses LLM judges to assess benchmark consistency, complexity, and policy coverage, while providing actionable diagnostics of weaknesses. We validate the framework by demonstrating agreement with independent human annotations and by evaluating benchmarks generated by LLMs of var","authors_text":"Abigail Goldsteen, Noam Koren, Roy Bar-Haim","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2608.06329","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e0b53ac7974e2f3189142525886717cace455bca7bd9a661851a6626bd54eea1","target":"record","created_at":"2026-08-07T01:41:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"521dba05ed6bf22d8571d21f417abcce37166426e5e95b0f100751c740bf4d2a","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2026-08-06T17:39:21Z","title_canon_sha256":"b6940799c67787c610d525435804dce867564d5f23638706170efc0acf3473e0"},"schema_version":"1.0","source":{"id":"2608.06329","kind":"arxiv","version":1}},"canonical_sha256":"3c8be5160f542a0442032cde7fdb8944c554fd4aa45599d2165f1b14d8b33b46","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3c8be5160f542a0442032cde7fdb8944c554fd4aa45599d2165f1b14d8b33b46","first_computed_at":"2026-08-07T01:41:22.789123Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-08-07T01:41:22.789123Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ZtdYR77cQvDCbuzD/CBR4kHMtroMKI54ra02XYZKouWe+BNcJtRATpKJSWQiSUkdEOouGgipfIcstn+UTAVACg==","signature_status":"signed_v1","signed_at":"2026-08-07T01:41:22.791056Z","signed_message":"canonical_sha256_bytes"},"source_id":"2608.06329","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e0b53ac7974e2f3189142525886717cace455bca7bd9a661851a6626bd54eea1","sha256:214068e1903e2ae141203fe7fecb0953adadf7f50e90adb58d778fc8683b469c"],"state_sha256":"5632ae68c6f882df1bae80003a9988ad2fb8b0ca1a7c11fc53c6a07a29373324"}