{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:BJHVWC3FS7LPO632JQKA7EU5GL","short_pith_number":"pith:BJHVWC3F","canonical_record":{"source":{"id":"2102.10242","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-02-20T03:29:20Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"dc56a8539ccdb271e6b2c02c00a5b9d79472821b5fe46eca92f86950b068fc1f","abstract_canon_sha256":"b57c7eec695e3bce409ac10b1538c37c5dd7719a8d25f4be8f3d50764a242156"},"schema_version":"1.0"},"canonical_sha256":"0a4f5b0b6597d6f77b7a4c140f929d32dd5ecf7627ae52bbceeb8d23784fc3ad","source":{"kind":"arxiv","id":"2102.10242","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.10242","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"arxiv_version","alias_value":"2102.10242v3","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.10242","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_12","alias_value":"BJHVWC3FS7LP","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_16","alias_value":"BJHVWC3FS7LPO632","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_8","alias_value":"BJHVWC3F","created_at":"2026-07-05T03:16:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:BJHVWC3FS7LPO632JQKA7EU5GL","target":"record","payload":{"canonical_record":{"source":{"id":"2102.10242","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-02-20T03:29:20Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"dc56a8539ccdb271e6b2c02c00a5b9d79472821b5fe46eca92f86950b068fc1f","abstract_canon_sha256":"b57c7eec695e3bce409ac10b1538c37c5dd7719a8d25f4be8f3d50764a242156"},"schema_version":"1.0"},"canonical_sha256":"0a4f5b0b6597d6f77b7a4c140f929d32dd5ecf7627ae52bbceeb8d23784fc3ad","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:16:31.147533Z","signature_b64":"7TXTqqoRf2m+zfTRvUa2ytNEZHMbNaNX4xhl+TuQx7ZpirO5KnssxOFzsbTG1FQOhHHKjshw5nXHI9o/IPYsDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0a4f5b0b6597d6f77b7a4c140f929d32dd5ecf7627ae52bbceeb8d23784fc3ad","last_reissued_at":"2026-07-05T03:16:31.147029Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:16:31.147029Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2102.10242","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:16:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"V7HvNr7oiySM3RIVWeewO2YrGuPHcJaZG5o3dADS37yDtfB03k7D+/lh0ezBO+vzXt1A61QjR6WlGeZ/kEwCBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T14:09:40.383044Z"},"content_sha256":"5051003feb98af653881af634822d2c6e9ce79702a8c3a2ba55fbb1b3a891234","schema_version":"1.0","event_id":"sha256:5051003feb98af653881af634822d2c6e9ce79702a8c3a2ba55fbb1b3a891234"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:BJHVWC3FS7LPO632JQKA7EU5GL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards Automatic Evaluation of Dialog Systems: A Model-Free Off-Policy Evaluation Approach","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Bo Dai, Haoming Jiang, Mengjiao Yang, Tuo Zhao, Wei Wei","submitted_at":"2021-02-20T03:29:20Z","abstract_excerpt":"Reliable automatic evaluation of dialogue systems under an interactive environment has long been overdue. An ideal environment for evaluating dialog systems, also known as the Turing test, needs to involve human interaction, which is usually not affordable for large-scale experiments. Though researchers have attempted to use metrics (e.g., perplexity, BLEU) in language generation tasks or some model-based reinforcement learning methods (e.g., self-play evaluation) for automatic evaluation, these methods only show a very weak correlation with the actual human evaluation in practice. To bridge s"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.10242","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2102.10242/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:16:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"530ncAezr5QpMUPYhylsWq8YoByOpfFESHMhxAyNeOhWZSas3V6ScxV62QfurPzAH4USgMak9sykcC6lhmuUCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T14:09:40.383442Z"},"content_sha256":"cda0e216f867c60be51565d2297d79f3a773bf7fa7cfc8fb46e98035b2614a4f","schema_version":"1.0","event_id":"sha256:cda0e216f867c60be51565d2297d79f3a773bf7fa7cfc8fb46e98035b2614a4f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BJHVWC3FS7LPO632JQKA7EU5GL/bundle.json","state_url":"https://pith.science/pith/BJHVWC3FS7LPO632JQKA7EU5GL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BJHVWC3FS7LPO632JQKA7EU5GL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-11T14:09:40Z","links":{"resolver":"https://pith.science/pith/BJHVWC3FS7LPO632JQKA7EU5GL","bundle":"https://pith.science/pith/BJHVWC3FS7LPO632JQKA7EU5GL/bundle.json","state":"https://pith.science/pith/BJHVWC3FS7LPO632JQKA7EU5GL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BJHVWC3FS7LPO632JQKA7EU5GL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:BJHVWC3FS7LPO632JQKA7EU5GL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b57c7eec695e3bce409ac10b1538c37c5dd7719a8d25f4be8f3d50764a242156","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-02-20T03:29:20Z","title_canon_sha256":"dc56a8539ccdb271e6b2c02c00a5b9d79472821b5fe46eca92f86950b068fc1f"},"schema_version":"1.0","source":{"id":"2102.10242","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.10242","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"arxiv_version","alias_value":"2102.10242v3","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.10242","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_12","alias_value":"BJHVWC3FS7LP","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_16","alias_value":"BJHVWC3FS7LPO632","created_at":"2026-07-05T03:16:31Z"},{"alias_kind":"pith_short_8","alias_value":"BJHVWC3F","created_at":"2026-07-05T03:16:31Z"}],"graph_snapshots":[{"event_id":"sha256:cda0e216f867c60be51565d2297d79f3a773bf7fa7cfc8fb46e98035b2614a4f","target":"graph","created_at":"2026-07-05T03:16:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2102.10242/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reliable automatic evaluation of dialogue systems under an interactive environment has long been overdue. An ideal environment for evaluating dialog systems, also known as the Turing test, needs to involve human interaction, which is usually not affordable for large-scale experiments. Though researchers have attempted to use metrics (e.g., perplexity, BLEU) in language generation tasks or some model-based reinforcement learning methods (e.g., self-play evaluation) for automatic evaluation, these methods only show a very weak correlation with the actual human evaluation in practice. To bridge s","authors_text":"Bo Dai, Haoming Jiang, Mengjiao Yang, Tuo Zhao, Wei Wei","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-02-20T03:29:20Z","title":"Towards Automatic Evaluation of Dialog Systems: A Model-Free Off-Policy Evaluation Approach"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.10242","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5051003feb98af653881af634822d2c6e9ce79702a8c3a2ba55fbb1b3a891234","target":"record","created_at":"2026-07-05T03:16:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b57c7eec695e3bce409ac10b1538c37c5dd7719a8d25f4be8f3d50764a242156","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-02-20T03:29:20Z","title_canon_sha256":"dc56a8539ccdb271e6b2c02c00a5b9d79472821b5fe46eca92f86950b068fc1f"},"schema_version":"1.0","source":{"id":"2102.10242","kind":"arxiv","version":3}},"canonical_sha256":"0a4f5b0b6597d6f77b7a4c140f929d32dd5ecf7627ae52bbceeb8d23784fc3ad","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0a4f5b0b6597d6f77b7a4c140f929d32dd5ecf7627ae52bbceeb8d23784fc3ad","first_computed_at":"2026-07-05T03:16:31.147029Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:16:31.147029Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"7TXTqqoRf2m+zfTRvUa2ytNEZHMbNaNX4xhl+TuQx7ZpirO5KnssxOFzsbTG1FQOhHHKjshw5nXHI9o/IPYsDA==","signature_status":"signed_v1","signed_at":"2026-07-05T03:16:31.147533Z","signed_message":"canonical_sha256_bytes"},"source_id":"2102.10242","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5051003feb98af653881af634822d2c6e9ce79702a8c3a2ba55fbb1b3a891234","sha256:cda0e216f867c60be51565d2297d79f3a773bf7fa7cfc8fb46e98035b2614a4f"],"state_sha256":"68358a13b4ed8fd7de9fd3229a4fac4383b7dd8cc014d2f25fbcc16f09cf2cc1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i7hLNsbuPhz5ANJofJD52MeThcv6WEhP3oO4akC+ouEgW6vo/MqtogvOqp3K99oxRRStkrbsbGB6pIUcrmiRAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-11T14:09:40.385626Z","bundle_sha256":"bc10660fef503776a0756a70c689598f618f7df416346cea38ed3887ee1e75e0"}}