{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:3SOOTSL3UOBMDXTDQ4SMIFR7TN","short_pith_number":"pith:3SOOTSL3","canonical_record":{"source":{"id":"2210.14529","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-26T07:41:32Z","cross_cats_sorted":[],"title_canon_sha256":"10dc392b6d580d38c355225cbc9166d0336bb7b731410fc887f02ccfa8d85d4d","abstract_canon_sha256":"011ee6ef66cef4ebef87991f4f5b8b2d6f514bf37caeab5651c8e3c6a3b8ed84"},"schema_version":"1.0"},"canonical_sha256":"dc9ce9c97ba382c1de638724c4163f9b492bbb87f8b52c3e27be6925d73131e9","source":{"kind":"arxiv","id":"2210.14529","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.14529","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"arxiv_version","alias_value":"2210.14529v1","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.14529","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_12","alias_value":"3SOOTSL3UOBM","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_16","alias_value":"3SOOTSL3UOBMDXTD","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_8","alias_value":"3SOOTSL3","created_at":"2026-07-05T05:10:46Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:3SOOTSL3UOBMDXTDQ4SMIFR7TN","target":"record","payload":{"canonical_record":{"source":{"id":"2210.14529","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-26T07:41:32Z","cross_cats_sorted":[],"title_canon_sha256":"10dc392b6d580d38c355225cbc9166d0336bb7b731410fc887f02ccfa8d85d4d","abstract_canon_sha256":"011ee6ef66cef4ebef87991f4f5b8b2d6f514bf37caeab5651c8e3c6a3b8ed84"},"schema_version":"1.0"},"canonical_sha256":"dc9ce9c97ba382c1de638724c4163f9b492bbb87f8b52c3e27be6925d73131e9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:10:46.435201Z","signature_b64":"cN6rlskx782XkcHluNPbulym7/fayZ01tj4H3nUwyyT36H8M7ZMUxJpqeXQvLVY8LqW8+gIlLkaK+XUyOCVpBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dc9ce9c97ba382c1de638724c4163f9b492bbb87f8b52c3e27be6925d73131e9","last_reissued_at":"2026-07-05T05:10:46.434846Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:10:46.434846Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.14529","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VcUvTGP4Wg//aFa2KS/lA7QLy0Qs27YPmzVbu9DpXOG/0cIifOeacFkoA1LcVlXDcJs64B0X25l0cQLequ7FBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T23:11:49.293626Z"},"content_sha256":"3517ef5b08eebc1e124ee002fb41f2fa924165b20eb03b0ac4ed86eff3c384cd","schema_version":"1.0","event_id":"sha256:3517ef5b08eebc1e124ee002fb41f2fa924165b20eb03b0ac4ed86eff3c384cd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:3SOOTSL3UOBMDXTDQ4SMIFR7TN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Is MultiWOZ a Solved Task? An Interactive TOD Evaluation Framework with User Simulator","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Feng Gao, Guofeng Quan, Linyang Li, Qinyuan Cheng, Xiaofeng Mou, Xipeng Qiu","submitted_at":"2022-10-26T07:41:32Z","abstract_excerpt":"Task-Oriented Dialogue (TOD) systems are drawing more and more attention in recent studies. Current methods focus on constructing pre-trained models or fine-tuning strategies while the evaluation of TOD is limited by a policy mismatch problem. That is, during evaluation, the user utterances are from the annotated dataset while these utterances should interact with previous responses which can have many alternatives besides annotated texts. Therefore, in this work, we propose an interactive evaluation framework for TOD. We first build a goal-oriented user simulator based on pre-trained models a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.14529","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.14529/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MVRkMphiPJZPypJ0cRrPuDderRd8Ph0MUOr9SSAb7yaDUFRcmFa1X4btUPxGs5fGJhUZlXic0o8/USGEdV5vDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T23:11:49.294164Z"},"content_sha256":"96f6d3a8194dad7b9082e5fc2c0a1a5ae52db576a9116dfd91df454b89827c59","schema_version":"1.0","event_id":"sha256:96f6d3a8194dad7b9082e5fc2c0a1a5ae52db576a9116dfd91df454b89827c59"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/bundle.json","state_url":"https://pith.science/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T23:11:49Z","links":{"resolver":"https://pith.science/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN","bundle":"https://pith.science/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/bundle.json","state":"https://pith.science/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3SOOTSL3UOBMDXTDQ4SMIFR7TN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:3SOOTSL3UOBMDXTDQ4SMIFR7TN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"011ee6ef66cef4ebef87991f4f5b8b2d6f514bf37caeab5651c8e3c6a3b8ed84","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-26T07:41:32Z","title_canon_sha256":"10dc392b6d580d38c355225cbc9166d0336bb7b731410fc887f02ccfa8d85d4d"},"schema_version":"1.0","source":{"id":"2210.14529","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.14529","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"arxiv_version","alias_value":"2210.14529v1","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.14529","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_12","alias_value":"3SOOTSL3UOBM","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_16","alias_value":"3SOOTSL3UOBMDXTD","created_at":"2026-07-05T05:10:46Z"},{"alias_kind":"pith_short_8","alias_value":"3SOOTSL3","created_at":"2026-07-05T05:10:46Z"}],"graph_snapshots":[{"event_id":"sha256:96f6d3a8194dad7b9082e5fc2c0a1a5ae52db576a9116dfd91df454b89827c59","target":"graph","created_at":"2026-07-05T05:10:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.14529/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Task-Oriented Dialogue (TOD) systems are drawing more and more attention in recent studies. Current methods focus on constructing pre-trained models or fine-tuning strategies while the evaluation of TOD is limited by a policy mismatch problem. That is, during evaluation, the user utterances are from the annotated dataset while these utterances should interact with previous responses which can have many alternatives besides annotated texts. Therefore, in this work, we propose an interactive evaluation framework for TOD. We first build a goal-oriented user simulator based on pre-trained models a","authors_text":"Feng Gao, Guofeng Quan, Linyang Li, Qinyuan Cheng, Xiaofeng Mou, Xipeng Qiu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-26T07:41:32Z","title":"Is MultiWOZ a Solved Task? An Interactive TOD Evaluation Framework with User Simulator"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.14529","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3517ef5b08eebc1e124ee002fb41f2fa924165b20eb03b0ac4ed86eff3c384cd","target":"record","created_at":"2026-07-05T05:10:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"011ee6ef66cef4ebef87991f4f5b8b2d6f514bf37caeab5651c8e3c6a3b8ed84","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-26T07:41:32Z","title_canon_sha256":"10dc392b6d580d38c355225cbc9166d0336bb7b731410fc887f02ccfa8d85d4d"},"schema_version":"1.0","source":{"id":"2210.14529","kind":"arxiv","version":1}},"canonical_sha256":"dc9ce9c97ba382c1de638724c4163f9b492bbb87f8b52c3e27be6925d73131e9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dc9ce9c97ba382c1de638724c4163f9b492bbb87f8b52c3e27be6925d73131e9","first_computed_at":"2026-07-05T05:10:46.434846Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:10:46.434846Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cN6rlskx782XkcHluNPbulym7/fayZ01tj4H3nUwyyT36H8M7ZMUxJpqeXQvLVY8LqW8+gIlLkaK+XUyOCVpBg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:10:46.435201Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.14529","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3517ef5b08eebc1e124ee002fb41f2fa924165b20eb03b0ac4ed86eff3c384cd","sha256:96f6d3a8194dad7b9082e5fc2c0a1a5ae52db576a9116dfd91df454b89827c59"],"state_sha256":"e092917432809d6db9713b1e128c7c45bb8de1bfa7546122aa6e7432fa3bfaf3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"83zQ2F7CwRA71Jiyi2/A/zA365eiBzqBFYY5djH/UOFVqvTP89B9dcwJjO/jK5mH5ZdvrWrwTNhEA2b9vbO/AQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T23:11:49.297822Z","bundle_sha256":"77240bee91d68a938273ed7be7906f70696a49e4d669809c7472b92a4098c227"}}