{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:UHUJC4HRR7BF2UOOUMSXDED4S5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e2c4ecfe3f871b1b6afc76cf6a70f8ba2b978daeb530d1ad55e400b6f31fec9f","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-21T14:15:46Z","title_canon_sha256":"7b491d9d2ccc35a01f6ab1873c230df365a1aaf4d43fb07d5b2a4612a14dbfed"},"schema_version":"1.0","source":{"id":"2312.13871","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.13871","created_at":"2026-07-05T08:05:07Z"},{"alias_kind":"arxiv_version","alias_value":"2312.13871v2","created_at":"2026-07-05T08:05:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.13871","created_at":"2026-07-05T08:05:07Z"},{"alias_kind":"pith_short_12","alias_value":"UHUJC4HRR7BF","created_at":"2026-07-05T08:05:07Z"},{"alias_kind":"pith_short_16","alias_value":"UHUJC4HRR7BF2UOO","created_at":"2026-07-05T08:05:07Z"},{"alias_kind":"pith_short_8","alias_value":"UHUJC4HR","created_at":"2026-07-05T08:05:07Z"}],"graph_snapshots":[{"event_id":"sha256:6e2e09dec46bd42a19aca5579311b75ed6366407619fd649e51f559fcc6ce551","target":"graph","created_at":"2026-07-05T08:05:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.13871/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This review gives an extensive overview of evaluation methods for task-oriented dialogue systems, paying special attention to practical applications of dialogue systems, for example for customer service. The review (1) provides an overview of the used constructs and metrics in previous work, (2) discusses challenges in the context of dialogue system evaluation and (3) develops a research agenda for the future of dialogue system evaluation. We conducted a systematic review of four databases (ACL, ACM, IEEE and Web of Science), which after screening resulted in 122 studies. Those studies were ca","authors_text":"Anouck Braggaar, Christine Liebrecht, Emiel Krahmer, Emiel van Miltenburg","cross_cats":["cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-21T14:15:46Z","title":"Evaluating Task-oriented Dialogue Systems: A Systematic Review of Measures, Constructs and their Operationalisations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.13871","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:18fcca9e4bdffab49b4f1246c83f65f308c05342f418f716bd4ad705e14273fb","target":"record","created_at":"2026-07-05T08:05:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e2c4ecfe3f871b1b6afc76cf6a70f8ba2b978daeb530d1ad55e400b6f31fec9f","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-12-21T14:15:46Z","title_canon_sha256":"7b491d9d2ccc35a01f6ab1873c230df365a1aaf4d43fb07d5b2a4612a14dbfed"},"schema_version":"1.0","source":{"id":"2312.13871","kind":"arxiv","version":2}},"canonical_sha256":"a1e89170f18fc25d51cea32571907c976714cf8ff30f6ef4ac4a22e902af2e2f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a1e89170f18fc25d51cea32571907c976714cf8ff30f6ef4ac4a22e902af2e2f","first_computed_at":"2026-07-05T08:05:07.484809Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:05:07.484809Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LVF/R9cPMqkySNVUSicR4t/+xLkN8sznK6b7vuc8ZJsVxF30LwyqvKgxQTZzVLams84HwL4l3e1p/pdnlE04AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:05:07.485223Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.13871","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:18fcca9e4bdffab49b4f1246c83f65f308c05342f418f716bd4ad705e14273fb","sha256:6e2e09dec46bd42a19aca5579311b75ed6366407619fd649e51f559fcc6ce551"],"state_sha256":"6db7289ef438c51bd4b6b0afc9efe4c35ca7f8894200ac8b18baae68e57619e1"}