{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:DQ2BGO4RPMT5IFBFSJHIKMDFNN","short_pith_number":"pith:DQ2BGO4R","canonical_record":{"source":{"id":"2309.02077","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T09:24:48Z","cross_cats_sorted":[],"title_canon_sha256":"f05bf7c730afafa2031251c231bc6f027a621c6b74f7be2914f70118a03b592b","abstract_canon_sha256":"61501203eb9fd34a8648e31a422a164056212f47fc189b348cc0ba7a26fee710"},"schema_version":"1.0"},"canonical_sha256":"1c34133b917b27d41425924e8530656b469d45aa83bd987b645bed5436b29efe","source":{"kind":"arxiv","id":"2309.02077","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.02077","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"arxiv_version","alias_value":"2309.02077v1","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.02077","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_12","alias_value":"DQ2BGO4RPMT5","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_16","alias_value":"DQ2BGO4RPMT5IFBF","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_8","alias_value":"DQ2BGO4R","created_at":"2026-07-05T06:47:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:DQ2BGO4RPMT5IFBFSJHIKMDFNN","target":"record","payload":{"canonical_record":{"source":{"id":"2309.02077","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T09:24:48Z","cross_cats_sorted":[],"title_canon_sha256":"f05bf7c730afafa2031251c231bc6f027a621c6b74f7be2914f70118a03b592b","abstract_canon_sha256":"61501203eb9fd34a8648e31a422a164056212f47fc189b348cc0ba7a26fee710"},"schema_version":"1.0"},"canonical_sha256":"1c34133b917b27d41425924e8530656b469d45aa83bd987b645bed5436b29efe","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:47:47.570945Z","signature_b64":"kbcNvxfzMPFkGo51jhKb5UcIcY76ZL/L9VgS3lxOl3TInT18HuZcphkaGhbEXh5HwjO8FVgqo+7PkcWBZBE/BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1c34133b917b27d41425924e8530656b469d45aa83bd987b645bed5436b29efe","last_reissued_at":"2026-07-05T06:47:47.570360Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:47:47.570360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.02077","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:47:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ycZtkSzs3+UcZ+sxn29l9OEBFsrOpi9ja9pYjiqfydfEQ1AALxOMnRsgTaQ9BElNn1jNyu8PKhbdzWyEnbdHAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-19T10:15:01.920724Z"},"content_sha256":"2df2c9379b8000d4ed0102f1485a2360e608c7d81e2cf2c74625dfc457cdfce1","schema_version":"1.0","event_id":"sha256:2df2c9379b8000d4ed0102f1485a2360e608c7d81e2cf2c74625dfc457cdfce1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:DQ2BGO4RPMT5IFBFSJHIKMDFNN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An Automatic Evaluation Framework for Multi-turn Medical Consultations Capabilities of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hongcheng Liu, Yanfeng Wang, Yusheng Liao, Yutong Meng, Yu Wang","submitted_at":"2023-09-05T09:24:48Z","abstract_excerpt":"Large language models (LLMs) have achieved significant success in interacting with human. However, recent studies have revealed that these models often suffer from hallucinations, leading to overly confident but incorrect judgments. This limits their application in the medical domain, where tasks require the utmost accuracy. This paper introduces an automated evaluation framework that assesses the practical capabilities of LLMs as virtual doctors during multi-turn consultations. Consultation tasks are designed to require LLMs to be aware of what they do not know, to inquire about missing medic"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.02077","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.02077/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:47:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k87G8pgdRufaAKYj57UAvOThEULd0YdhHQ5/VTnP9MmZCpQPo9wLpThqe9X/9e3W8YMJE2Ea+rQYTwH2pq2hBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-19T10:15:01.921324Z"},"content_sha256":"dce8412a63e2171e7b87c79c5ee59ff6fb8f23f5e0b9d159604134240c208694","schema_version":"1.0","event_id":"sha256:dce8412a63e2171e7b87c79c5ee59ff6fb8f23f5e0b9d159604134240c208694"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/bundle.json","state_url":"https://pith.science/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-19T10:15:01Z","links":{"resolver":"https://pith.science/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN","bundle":"https://pith.science/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/bundle.json","state":"https://pith.science/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DQ2BGO4RPMT5IFBFSJHIKMDFNN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:DQ2BGO4RPMT5IFBFSJHIKMDFNN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"61501203eb9fd34a8648e31a422a164056212f47fc189b348cc0ba7a26fee710","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T09:24:48Z","title_canon_sha256":"f05bf7c730afafa2031251c231bc6f027a621c6b74f7be2914f70118a03b592b"},"schema_version":"1.0","source":{"id":"2309.02077","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.02077","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"arxiv_version","alias_value":"2309.02077v1","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.02077","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_12","alias_value":"DQ2BGO4RPMT5","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_16","alias_value":"DQ2BGO4RPMT5IFBF","created_at":"2026-07-05T06:47:47Z"},{"alias_kind":"pith_short_8","alias_value":"DQ2BGO4R","created_at":"2026-07-05T06:47:47Z"}],"graph_snapshots":[{"event_id":"sha256:dce8412a63e2171e7b87c79c5ee59ff6fb8f23f5e0b9d159604134240c208694","target":"graph","created_at":"2026-07-05T06:47:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.02077/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have achieved significant success in interacting with human. However, recent studies have revealed that these models often suffer from hallucinations, leading to overly confident but incorrect judgments. This limits their application in the medical domain, where tasks require the utmost accuracy. This paper introduces an automated evaluation framework that assesses the practical capabilities of LLMs as virtual doctors during multi-turn consultations. Consultation tasks are designed to require LLMs to be aware of what they do not know, to inquire about missing medic","authors_text":"Hongcheng Liu, Yanfeng Wang, Yusheng Liao, Yutong Meng, Yu Wang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T09:24:48Z","title":"An Automatic Evaluation Framework for Multi-turn Medical Consultations Capabilities of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.02077","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2df2c9379b8000d4ed0102f1485a2360e608c7d81e2cf2c74625dfc457cdfce1","target":"record","created_at":"2026-07-05T06:47:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"61501203eb9fd34a8648e31a422a164056212f47fc189b348cc0ba7a26fee710","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-05T09:24:48Z","title_canon_sha256":"f05bf7c730afafa2031251c231bc6f027a621c6b74f7be2914f70118a03b592b"},"schema_version":"1.0","source":{"id":"2309.02077","kind":"arxiv","version":1}},"canonical_sha256":"1c34133b917b27d41425924e8530656b469d45aa83bd987b645bed5436b29efe","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1c34133b917b27d41425924e8530656b469d45aa83bd987b645bed5436b29efe","first_computed_at":"2026-07-05T06:47:47.570360Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:47:47.570360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kbcNvxfzMPFkGo51jhKb5UcIcY76ZL/L9VgS3lxOl3TInT18HuZcphkaGhbEXh5HwjO8FVgqo+7PkcWBZBE/BQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:47:47.570945Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.02077","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2df2c9379b8000d4ed0102f1485a2360e608c7d81e2cf2c74625dfc457cdfce1","sha256:dce8412a63e2171e7b87c79c5ee59ff6fb8f23f5e0b9d159604134240c208694"],"state_sha256":"ded68a150aa6a6550c991f78f3930c88e4151be8ca7446f3a56d420db97da118"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HtnV1Gtj/SoLTrbGRS2tqMQwRBIHpqQsqTA/jGcd9tddBwaktorwhygGf9dFqFdIxbb0Jhda7exQ3Cx2te61AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-19T10:15:01.924755Z","bundle_sha256":"79188a9b4cdf773c18ca22d63d12ef684634f7e01abe6c5423c0e2ee04a1c47e"}}