{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:GAKHJQIBRH6TE4ZJY5GUD7DHWH","short_pith_number":"pith:GAKHJQIB","canonical_record":{"source":{"id":"2106.03706","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-06-07T15:17:03Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"855ef3d690fc33bd1b994c1119b48284ab997c52b89bb34b08fcc24cfe214aea","abstract_canon_sha256":"a113e50af81b44589f1320e7d41638f8cc4a400cc79edddb76ed1c7824d36364"},"schema_version":"1.0"},"canonical_sha256":"301474c10189fd327329c74d41fc67b1d158ae952ed49a96ed4923da8c09e854","source":{"kind":"arxiv","id":"2106.03706","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.03706","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"arxiv_version","alias_value":"2106.03706v4","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.03706","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_12","alias_value":"GAKHJQIBRH6T","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_16","alias_value":"GAKHJQIBRH6TE4ZJ","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_8","alias_value":"GAKHJQIB","created_at":"2026-07-05T02:56:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:GAKHJQIBRH6TE4ZJY5GUD7DHWH","target":"record","payload":{"canonical_record":{"source":{"id":"2106.03706","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-06-07T15:17:03Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"855ef3d690fc33bd1b994c1119b48284ab997c52b89bb34b08fcc24cfe214aea","abstract_canon_sha256":"a113e50af81b44589f1320e7d41638f8cc4a400cc79edddb76ed1c7824d36364"},"schema_version":"1.0"},"canonical_sha256":"301474c10189fd327329c74d41fc67b1d158ae952ed49a96ed4923da8c09e854","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:56:05.042005Z","signature_b64":"Lxr8LvqqZw6l13DfyzS4NYvQOVbZ9xa/qaFqytvoluS5qY9v0neCR96P2vGFpfYOgokwHa2uN8Hvdva58JYDBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"301474c10189fd327329c74d41fc67b1d158ae952ed49a96ed4923da8c09e854","last_reissued_at":"2026-07-05T02:56:05.041596Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:56:05.041596Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2106.03706","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:56:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8qShZtz/Zx4Ef5iTFmuk90f1hgrs8HB78S3N7mmA7lTlNWKR+LuqtCJ6vEfmoiSGU4vPNcI+g/3ejwwob2+8DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T22:03:53.210647Z"},"content_sha256":"2ca66f7c986349a728bfe1716b8c5404f9ac45d9f7a2dd043c181b534b4f0986","schema_version":"1.0","event_id":"sha256:2ca66f7c986349a728bfe1716b8c5404f9ac45d9f7a2dd043c181b534b4f0986"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:GAKHJQIBRH6TE4ZJY5GUD7DHWH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Comprehensive Assessment of Dialog Evaluation Metrics","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Maxine Eskenazi, Shikib Mehri, Yi-Ting Yeh","submitted_at":"2021-06-07T15:17:03Z","abstract_excerpt":"Automatic evaluation metrics are a crucial component of dialog systems research. Standard language evaluation metrics are known to be ineffective for evaluating dialog. As such, recent research has proposed a number of novel, dialog-specific metrics that correlate better with human judgements. Due to the fast pace of research, many of these metrics have been assessed on different datasets and there has as yet been no time for a systematic comparison between them. To this end, this paper provides a comprehensive assessment of recently proposed dialog evaluation metrics on a number of datasets. "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.03706","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.03706/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:56:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jymcf0WTrn68KJ5RpMdhfKxk3PRGep2wKim6bSUyLSEP7fsUa79lN3h61gyrwEIOzEYdL+YEC3NNtvLw9CLvAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T22:03:53.211174Z"},"content_sha256":"9f1cf1f972ac5b3f1b411671f9ded092f8eab8043751a12266c7ce203c0201b7","schema_version":"1.0","event_id":"sha256:9f1cf1f972ac5b3f1b411671f9ded092f8eab8043751a12266c7ce203c0201b7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/bundle.json","state_url":"https://pith.science/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T22:03:53Z","links":{"resolver":"https://pith.science/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH","bundle":"https://pith.science/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/bundle.json","state":"https://pith.science/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GAKHJQIBRH6TE4ZJY5GUD7DHWH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:GAKHJQIBRH6TE4ZJY5GUD7DHWH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a113e50af81b44589f1320e7d41638f8cc4a400cc79edddb76ed1c7824d36364","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-06-07T15:17:03Z","title_canon_sha256":"855ef3d690fc33bd1b994c1119b48284ab997c52b89bb34b08fcc24cfe214aea"},"schema_version":"1.0","source":{"id":"2106.03706","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.03706","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"arxiv_version","alias_value":"2106.03706v4","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.03706","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_12","alias_value":"GAKHJQIBRH6T","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_16","alias_value":"GAKHJQIBRH6TE4ZJ","created_at":"2026-07-05T02:56:05Z"},{"alias_kind":"pith_short_8","alias_value":"GAKHJQIB","created_at":"2026-07-05T02:56:05Z"}],"graph_snapshots":[{"event_id":"sha256:9f1cf1f972ac5b3f1b411671f9ded092f8eab8043751a12266c7ce203c0201b7","target":"graph","created_at":"2026-07-05T02:56:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2106.03706/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic evaluation metrics are a crucial component of dialog systems research. Standard language evaluation metrics are known to be ineffective for evaluating dialog. As such, recent research has proposed a number of novel, dialog-specific metrics that correlate better with human judgements. Due to the fast pace of research, many of these metrics have been assessed on different datasets and there has as yet been no time for a systematic comparison between them. To this end, this paper provides a comprehensive assessment of recently proposed dialog evaluation metrics on a number of datasets. ","authors_text":"Maxine Eskenazi, Shikib Mehri, Yi-Ting Yeh","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-06-07T15:17:03Z","title":"A Comprehensive Assessment of Dialog Evaluation Metrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.03706","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2ca66f7c986349a728bfe1716b8c5404f9ac45d9f7a2dd043c181b534b4f0986","target":"record","created_at":"2026-07-05T02:56:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a113e50af81b44589f1320e7d41638f8cc4a400cc79edddb76ed1c7824d36364","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-06-07T15:17:03Z","title_canon_sha256":"855ef3d690fc33bd1b994c1119b48284ab997c52b89bb34b08fcc24cfe214aea"},"schema_version":"1.0","source":{"id":"2106.03706","kind":"arxiv","version":4}},"canonical_sha256":"301474c10189fd327329c74d41fc67b1d158ae952ed49a96ed4923da8c09e854","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"301474c10189fd327329c74d41fc67b1d158ae952ed49a96ed4923da8c09e854","first_computed_at":"2026-07-05T02:56:05.041596Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:56:05.041596Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Lxr8LvqqZw6l13DfyzS4NYvQOVbZ9xa/qaFqytvoluS5qY9v0neCR96P2vGFpfYOgokwHa2uN8Hvdva58JYDBA==","signature_status":"signed_v1","signed_at":"2026-07-05T02:56:05.042005Z","signed_message":"canonical_sha256_bytes"},"source_id":"2106.03706","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2ca66f7c986349a728bfe1716b8c5404f9ac45d9f7a2dd043c181b534b4f0986","sha256:9f1cf1f972ac5b3f1b411671f9ded092f8eab8043751a12266c7ce203c0201b7"],"state_sha256":"1d14e04c6bee23f558d4f2cad96b8dfe5b5727ef05c274d26a907d69e85e67fd"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2JWEoi5ncjapiiXk/gaD6/KFWyO/ezLjy00F+e4e9NthbQ3VmF8OKwyp1a5cDuhPw8IkhH2bazEu1NN3hvrSDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T22:03:53.215394Z","bundle_sha256":"c3d15c7b7a538b0bc7594c2d71ab96fbb0feb241888194c90afe57e71c733363"}}