{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:IHM6STAKKXIQ4HANKFSEPCMV6U","short_pith_number":"pith:IHM6STAK","canonical_record":{"source":{"id":"2305.13693","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-23T05:00:59Z","cross_cats_sorted":[],"title_canon_sha256":"968b60c71ba541b6bbb86182d17c85f3858d28e9f9d3150456c1734389a1fb51","abstract_canon_sha256":"09e2326fc79fe6c873e514342048217f23ff699306a2b39bd68ab8ff735cb779"},"schema_version":"1.0"},"canonical_sha256":"41d9e94c0a55d10e1c0d5164478995f537a5733d448dcfd4eeb3e0e145c5dc03","source":{"kind":"arxiv","id":"2305.13693","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13693","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13693v1","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13693","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_12","alias_value":"IHM6STAKKXIQ","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_16","alias_value":"IHM6STAKKXIQ4HAN","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_8","alias_value":"IHM6STAK","created_at":"2026-07-05T06:12:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:IHM6STAKKXIQ4HANKFSEPCMV6U","target":"record","payload":{"canonical_record":{"source":{"id":"2305.13693","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-23T05:00:59Z","cross_cats_sorted":[],"title_canon_sha256":"968b60c71ba541b6bbb86182d17c85f3858d28e9f9d3150456c1734389a1fb51","abstract_canon_sha256":"09e2326fc79fe6c873e514342048217f23ff699306a2b39bd68ab8ff735cb779"},"schema_version":"1.0"},"canonical_sha256":"41d9e94c0a55d10e1c0d5164478995f537a5733d448dcfd4eeb3e0e145c5dc03","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:12:56.173204Z","signature_b64":"4Q6u6vQRDucG1mCNJizQ+hr7WF4oKoZzdPbQL1XmojZNsV00TfVGuqeEoPtwk8XNWtLlCTXcmjn1Cvfz8AZoCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"41d9e94c0a55d10e1c0d5164478995f537a5733d448dcfd4eeb3e0e145c5dc03","last_reissued_at":"2026-07-05T06:12:56.172874Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:12:56.172874Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.13693","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oDyA9E/LMvZ6POdNoaPPspV4d1SiXGfWgLeS4DBhOhBzf6N4DgF+osARdsH5k4ezYbwG7WaJJV/eMLEPRlFHAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T06:49:27.969661Z"},"content_sha256":"3ce16bcde219928c31c80542c35483115d4e3ea39afb186ec07527856f2441ee","schema_version":"1.0","event_id":"sha256:3ce16bcde219928c31c80542c35483115d4e3ea39afb186ec07527856f2441ee"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:IHM6STAKKXIQ4HANKFSEPCMV6U","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automated Metrics for Medical Multi-Document Summarization Disagree with Human Evaluations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Bailey E. Kuehl, Byron C. Wallace, Erin Bransom, Jay DeYoung, Lucy Lu Wang, Thinh Hung Truong, Yulia Otmakhova","submitted_at":"2023-05-23T05:00:59Z","abstract_excerpt":"Evaluating multi-document summarization (MDS) quality is difficult. This is especially true in the case of MDS for biomedical literature reviews, where models must synthesize contradicting evidence reported across different documents. Prior work has shown that rather than performing the task, models may exploit shortcuts that are difficult to detect using standard n-gram similarity metrics such as ROUGE. Better automated evaluation metrics are needed, but few resources exist to assess metrics when they are proposed. Therefore, we introduce a dataset of human-assessed summary quality facets and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13693","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.13693/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pJ9KOdL4625Zmh4J5VJ8LcbHMell1iKo6k/J0lO7+Z81g4qq2mJ7N7XPZ11RdnDf/j+2xpzxJPZuGnGQtTVvBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T06:49:27.970263Z"},"content_sha256":"8d2a03a18e9963228de741b417513632add542a0cff9efeecef73e7ccf25d67b","schema_version":"1.0","event_id":"sha256:8d2a03a18e9963228de741b417513632add542a0cff9efeecef73e7ccf25d67b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/bundle.json","state_url":"https://pith.science/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T06:49:27Z","links":{"resolver":"https://pith.science/pith/IHM6STAKKXIQ4HANKFSEPCMV6U","bundle":"https://pith.science/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/bundle.json","state":"https://pith.science/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IHM6STAKKXIQ4HANKFSEPCMV6U/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:IHM6STAKKXIQ4HANKFSEPCMV6U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"09e2326fc79fe6c873e514342048217f23ff699306a2b39bd68ab8ff735cb779","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-23T05:00:59Z","title_canon_sha256":"968b60c71ba541b6bbb86182d17c85f3858d28e9f9d3150456c1734389a1fb51"},"schema_version":"1.0","source":{"id":"2305.13693","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13693","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13693v1","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13693","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_12","alias_value":"IHM6STAKKXIQ","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_16","alias_value":"IHM6STAKKXIQ4HAN","created_at":"2026-07-05T06:12:56Z"},{"alias_kind":"pith_short_8","alias_value":"IHM6STAK","created_at":"2026-07-05T06:12:56Z"}],"graph_snapshots":[{"event_id":"sha256:8d2a03a18e9963228de741b417513632add542a0cff9efeecef73e7ccf25d67b","target":"graph","created_at":"2026-07-05T06:12:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.13693/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating multi-document summarization (MDS) quality is difficult. This is especially true in the case of MDS for biomedical literature reviews, where models must synthesize contradicting evidence reported across different documents. Prior work has shown that rather than performing the task, models may exploit shortcuts that are difficult to detect using standard n-gram similarity metrics such as ROUGE. Better automated evaluation metrics are needed, but few resources exist to assess metrics when they are proposed. Therefore, we introduce a dataset of human-assessed summary quality facets and","authors_text":"Bailey E. Kuehl, Byron C. Wallace, Erin Bransom, Jay DeYoung, Lucy Lu Wang, Thinh Hung Truong, Yulia Otmakhova","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-23T05:00:59Z","title":"Automated Metrics for Medical Multi-Document Summarization Disagree with Human Evaluations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13693","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3ce16bcde219928c31c80542c35483115d4e3ea39afb186ec07527856f2441ee","target":"record","created_at":"2026-07-05T06:12:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"09e2326fc79fe6c873e514342048217f23ff699306a2b39bd68ab8ff735cb779","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-23T05:00:59Z","title_canon_sha256":"968b60c71ba541b6bbb86182d17c85f3858d28e9f9d3150456c1734389a1fb51"},"schema_version":"1.0","source":{"id":"2305.13693","kind":"arxiv","version":1}},"canonical_sha256":"41d9e94c0a55d10e1c0d5164478995f537a5733d448dcfd4eeb3e0e145c5dc03","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"41d9e94c0a55d10e1c0d5164478995f537a5733d448dcfd4eeb3e0e145c5dc03","first_computed_at":"2026-07-05T06:12:56.172874Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:12:56.172874Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4Q6u6vQRDucG1mCNJizQ+hr7WF4oKoZzdPbQL1XmojZNsV00TfVGuqeEoPtwk8XNWtLlCTXcmjn1Cvfz8AZoCg==","signature_status":"signed_v1","signed_at":"2026-07-05T06:12:56.173204Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.13693","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3ce16bcde219928c31c80542c35483115d4e3ea39afb186ec07527856f2441ee","sha256:8d2a03a18e9963228de741b417513632add542a0cff9efeecef73e7ccf25d67b"],"state_sha256":"b9539355119e1bd7217685dd9304e8a942a8663c37ffba9431b1de9139f74726"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N7cP5FsOQxkz6DG4IGbMJGu3JW8wprryg1hp9n6g3kqQZAl/TZHWkpvDtz9BJCVo/SPtP2V2JGJtrFWL27DhDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T06:49:27.974721Z","bundle_sha256":"d4cf05ef4d7283d7dc078bf17d86f2d99ed4a57b3e75bc3c8f14193262870f10"}}