{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:JJCAMTDR7UOIYXMI3672R5E2VP","short_pith_number":"pith:JJCAMTDR","canonical_record":{"source":{"id":"2108.01369","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-03T08:52:33Z","cross_cats_sorted":[],"title_canon_sha256":"5714d7262f0bf2cf0ce5ce17ecc8c900fbbc10fd8b6123c626987bd64bbad207","abstract_canon_sha256":"fe4c26b18500a0e34cf84250873eb50dd16c9291ebe7b5a573c5079029e453a0"},"schema_version":"1.0"},"canonical_sha256":"4a44064c71fd1c8c5d88dfbfa8f49aabde125c1829921fcb4656c4a2c2aae5ce","source":{"kind":"arxiv","id":"2108.01369","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2108.01369","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"arxiv_version","alias_value":"2108.01369v1","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.01369","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_12","alias_value":"JJCAMTDR7UOI","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_16","alias_value":"JJCAMTDR7UOIYXMI","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_8","alias_value":"JJCAMTDR","created_at":"2026-07-05T03:03:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:JJCAMTDR7UOIYXMI3672R5E2VP","target":"record","payload":{"canonical_record":{"source":{"id":"2108.01369","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-03T08:52:33Z","cross_cats_sorted":[],"title_canon_sha256":"5714d7262f0bf2cf0ce5ce17ecc8c900fbbc10fd8b6123c626987bd64bbad207","abstract_canon_sha256":"fe4c26b18500a0e34cf84250873eb50dd16c9291ebe7b5a573c5079029e453a0"},"schema_version":"1.0"},"canonical_sha256":"4a44064c71fd1c8c5d88dfbfa8f49aabde125c1829921fcb4656c4a2c2aae5ce","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:03:02.000133Z","signature_b64":"Pq27mBZb0RrDGQMq9A8IVg+36R37dGjjkFhIMMfYdMR03gQ1rYdbEIlCkJMNVDpXVoaV2H6deZ4TAJsNITbHCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4a44064c71fd1c8c5d88dfbfa8f49aabde125c1829921fcb4656c4a2c2aae5ce","last_reissued_at":"2026-07-05T03:03:01.999657Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:03:01.999657Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2108.01369","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:03:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qMA6+ly9pkCm3CipHG8tkqlY9NJcJ4Wy7y9jp4sr6IIL62eklWkZPJBYI082FUereftKCHxSPwiEKWR02i0zDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T21:05:25.751901Z"},"content_sha256":"2c94fad9862178c5d2950760d105455673227d13f1b6c5dd509948c1f06d7e28","schema_version":"1.0","event_id":"sha256:2c94fad9862178c5d2950760d105455673227d13f1b6c5dd509948c1f06d7e28"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:JJCAMTDR7UOIYXMI3672R5E2VP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How to Evaluate Your Dialogue Models: A Review of Approaches","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Long Qin, Quanjun Yin, Wansen Wu, Xinmeng Li","submitted_at":"2021-08-03T08:52:33Z","abstract_excerpt":"Evaluating the quality of a dialogue system is an understudied problem. The recent evolution of evaluation method motivated this survey, in which an explicit and comprehensive analysis of the existing methods is sought. We are first to divide the evaluation methods into three classes, i.e., automatic evaluation, human-involved evaluation and user simulator based evaluation. Then, each class is covered with main features and the related evaluation metrics. The existence of benchmarks, suitable for the evaluation of dialogue techniques are also discussed in detail. Finally, some open issues are "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.01369","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2108.01369/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:03:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gxuoAMc4xkfns/9Kzfbd+EP22gDg6IgRZ4jVDMgKcPmPSCQ+cggAr1Q7HY6jwqRWQJ2M43UQwr/KDGo06t7IDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T21:05:25.752413Z"},"content_sha256":"06abb9788856e677621914ef5c9d2e12c75e5f4027e376e4329c931a34c3eb29","schema_version":"1.0","event_id":"sha256:06abb9788856e677621914ef5c9d2e12c75e5f4027e376e4329c931a34c3eb29"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JJCAMTDR7UOIYXMI3672R5E2VP/bundle.json","state_url":"https://pith.science/pith/JJCAMTDR7UOIYXMI3672R5E2VP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JJCAMTDR7UOIYXMI3672R5E2VP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T21:05:25Z","links":{"resolver":"https://pith.science/pith/JJCAMTDR7UOIYXMI3672R5E2VP","bundle":"https://pith.science/pith/JJCAMTDR7UOIYXMI3672R5E2VP/bundle.json","state":"https://pith.science/pith/JJCAMTDR7UOIYXMI3672R5E2VP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JJCAMTDR7UOIYXMI3672R5E2VP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:JJCAMTDR7UOIYXMI3672R5E2VP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fe4c26b18500a0e34cf84250873eb50dd16c9291ebe7b5a573c5079029e453a0","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-03T08:52:33Z","title_canon_sha256":"5714d7262f0bf2cf0ce5ce17ecc8c900fbbc10fd8b6123c626987bd64bbad207"},"schema_version":"1.0","source":{"id":"2108.01369","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2108.01369","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"arxiv_version","alias_value":"2108.01369v1","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.01369","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_12","alias_value":"JJCAMTDR7UOI","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_16","alias_value":"JJCAMTDR7UOIYXMI","created_at":"2026-07-05T03:03:01Z"},{"alias_kind":"pith_short_8","alias_value":"JJCAMTDR","created_at":"2026-07-05T03:03:01Z"}],"graph_snapshots":[{"event_id":"sha256:06abb9788856e677621914ef5c9d2e12c75e5f4027e376e4329c931a34c3eb29","target":"graph","created_at":"2026-07-05T03:03:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2108.01369/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating the quality of a dialogue system is an understudied problem. The recent evolution of evaluation method motivated this survey, in which an explicit and comprehensive analysis of the existing methods is sought. We are first to divide the evaluation methods into three classes, i.e., automatic evaluation, human-involved evaluation and user simulator based evaluation. Then, each class is covered with main features and the related evaluation metrics. The existence of benchmarks, suitable for the evaluation of dialogue techniques are also discussed in detail. Finally, some open issues are ","authors_text":"Long Qin, Quanjun Yin, Wansen Wu, Xinmeng Li","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-03T08:52:33Z","title":"How to Evaluate Your Dialogue Models: A Review of Approaches"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.01369","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2c94fad9862178c5d2950760d105455673227d13f1b6c5dd509948c1f06d7e28","target":"record","created_at":"2026-07-05T03:03:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fe4c26b18500a0e34cf84250873eb50dd16c9291ebe7b5a573c5079029e453a0","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-03T08:52:33Z","title_canon_sha256":"5714d7262f0bf2cf0ce5ce17ecc8c900fbbc10fd8b6123c626987bd64bbad207"},"schema_version":"1.0","source":{"id":"2108.01369","kind":"arxiv","version":1}},"canonical_sha256":"4a44064c71fd1c8c5d88dfbfa8f49aabde125c1829921fcb4656c4a2c2aae5ce","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4a44064c71fd1c8c5d88dfbfa8f49aabde125c1829921fcb4656c4a2c2aae5ce","first_computed_at":"2026-07-05T03:03:01.999657Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:03:01.999657Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Pq27mBZb0RrDGQMq9A8IVg+36R37dGjjkFhIMMfYdMR03gQ1rYdbEIlCkJMNVDpXVoaV2H6deZ4TAJsNITbHCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T03:03:02.000133Z","signed_message":"canonical_sha256_bytes"},"source_id":"2108.01369","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2c94fad9862178c5d2950760d105455673227d13f1b6c5dd509948c1f06d7e28","sha256:06abb9788856e677621914ef5c9d2e12c75e5f4027e376e4329c931a34c3eb29"],"state_sha256":"9891bc8cc1e1b5b5d6b2ccb2f3c91e8ec956ad65c007923c5c5a32054c97835c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9F3Nn8JE4g3jv0T6lJBlx7dU7lVtctRtoob8Eiz0wDXWXY0ubgtYg5U4BBm7qu5h5rEhJszE7yaHa1MruTcvBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T21:05:25.757294Z","bundle_sha256":"8892196c168259db9dce4b458eff54f256a651d45b33f8048e92becea16f0ab3"}}