{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:WQVWCDFOL4B2ZN6ZNFOENHPSHM","short_pith_number":"pith:WQVWCDFO","canonical_record":{"source":{"id":"2204.10216","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-21T15:52:14Z","cross_cats_sorted":[],"title_canon_sha256":"0af4bb7d33287eac1bf369f2f7862b1b2c9522c951184ae44c3362cb565c3aed","abstract_canon_sha256":"a77513574ec05e549d90438532fd9c7eb364db533c6467a7a1536d0a8e679ec3"},"schema_version":"1.0"},"canonical_sha256":"b42b610cae5f03acb7d9695c469df23b184b219f5c20573623d0b63480fbd23c","source":{"kind":"arxiv","id":"2204.10216","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.10216","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"arxiv_version","alias_value":"2204.10216v1","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.10216","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_12","alias_value":"WQVWCDFOL4B2","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_16","alias_value":"WQVWCDFOL4B2ZN6Z","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_8","alias_value":"WQVWCDFO","created_at":"2026-07-05T04:16:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:WQVWCDFOL4B2ZN6ZNFOENHPSHM","target":"record","payload":{"canonical_record":{"source":{"id":"2204.10216","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-21T15:52:14Z","cross_cats_sorted":[],"title_canon_sha256":"0af4bb7d33287eac1bf369f2f7862b1b2c9522c951184ae44c3362cb565c3aed","abstract_canon_sha256":"a77513574ec05e549d90438532fd9c7eb364db533c6467a7a1536d0a8e679ec3"},"schema_version":"1.0"},"canonical_sha256":"b42b610cae5f03acb7d9695c469df23b184b219f5c20573623d0b63480fbd23c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:16:48.007931Z","signature_b64":"FfFDf9SbE/0h2T1LFnnEelbJtn590N9BS4eTc38YBcMl0Uq0k0MKCK4aliYIvMLPpAXHFrwyl67rDCUdwF2eBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b42b610cae5f03acb7d9695c469df23b184b219f5c20573623d0b63480fbd23c","last_reissued_at":"2026-07-05T04:16:48.007443Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:16:48.007443Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2204.10216","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:16:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"03yFjipJskTHu+HRMRGw/gmkJHijGemdNEXwDFrWt82Jud/3lba3U1zTxhf+n4w0Zdmc/qCLQvoeko3DV7qzCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T02:57:48.939694Z"},"content_sha256":"4bbc0045c8c95217752effabb5f4127b959399ea110ae7920df79f1adb91da4d","schema_version":"1.0","event_id":"sha256:4bbc0045c8c95217752effabb5f4127b959399ea110ae7920df79f1adb91da4d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:WQVWCDFOL4B2ZN6ZNFOENHPSHM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Re-Examining System-Level Correlations of Automatic Summarization Evaluation Metrics","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Daniel Deutsch, Dan Roth, Rotem Dror","submitted_at":"2022-04-21T15:52:14Z","abstract_excerpt":"How reliably an automatic summarization evaluation metric replicates human judgments of summary quality is quantified by system-level correlations. We identify two ways in which the definition of the system-level correlation is inconsistent with how metrics are used to evaluate systems in practice and propose changes to rectify this disconnect. First, we calculate the system score for an automatic metric using the full test set instead of the subset of summaries judged by humans, which is currently standard practice. We demonstrate how this small change leads to more precise estimates of syste"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.10216","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2204.10216/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:16:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Hl1olBTi8IFlMaqIevmBYMLn4FqMoCqOPa53jKLLWMItfB54UtAywGgRuyju8Sc0Jl56Kj+myfHhIWCZOU5wDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T02:57:48.940250Z"},"content_sha256":"621ebcc3fd20f2864811a8990dc9efaa0e40d88e7a942a5d31bcf407049ea17a","schema_version":"1.0","event_id":"sha256:621ebcc3fd20f2864811a8990dc9efaa0e40d88e7a942a5d31bcf407049ea17a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/bundle.json","state_url":"https://pith.science/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T02:57:48Z","links":{"resolver":"https://pith.science/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM","bundle":"https://pith.science/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/bundle.json","state":"https://pith.science/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WQVWCDFOL4B2ZN6ZNFOENHPSHM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:WQVWCDFOL4B2ZN6ZNFOENHPSHM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a77513574ec05e549d90438532fd9c7eb364db533c6467a7a1536d0a8e679ec3","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-21T15:52:14Z","title_canon_sha256":"0af4bb7d33287eac1bf369f2f7862b1b2c9522c951184ae44c3362cb565c3aed"},"schema_version":"1.0","source":{"id":"2204.10216","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.10216","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"arxiv_version","alias_value":"2204.10216v1","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.10216","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_12","alias_value":"WQVWCDFOL4B2","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_16","alias_value":"WQVWCDFOL4B2ZN6Z","created_at":"2026-07-05T04:16:48Z"},{"alias_kind":"pith_short_8","alias_value":"WQVWCDFO","created_at":"2026-07-05T04:16:48Z"}],"graph_snapshots":[{"event_id":"sha256:621ebcc3fd20f2864811a8990dc9efaa0e40d88e7a942a5d31bcf407049ea17a","target":"graph","created_at":"2026-07-05T04:16:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2204.10216/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"How reliably an automatic summarization evaluation metric replicates human judgments of summary quality is quantified by system-level correlations. We identify two ways in which the definition of the system-level correlation is inconsistent with how metrics are used to evaluate systems in practice and propose changes to rectify this disconnect. First, we calculate the system score for an automatic metric using the full test set instead of the subset of summaries judged by humans, which is currently standard practice. We demonstrate how this small change leads to more precise estimates of syste","authors_text":"Daniel Deutsch, Dan Roth, Rotem Dror","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-21T15:52:14Z","title":"Re-Examining System-Level Correlations of Automatic Summarization Evaluation Metrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.10216","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4bbc0045c8c95217752effabb5f4127b959399ea110ae7920df79f1adb91da4d","target":"record","created_at":"2026-07-05T04:16:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a77513574ec05e549d90438532fd9c7eb364db533c6467a7a1536d0a8e679ec3","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-21T15:52:14Z","title_canon_sha256":"0af4bb7d33287eac1bf369f2f7862b1b2c9522c951184ae44c3362cb565c3aed"},"schema_version":"1.0","source":{"id":"2204.10216","kind":"arxiv","version":1}},"canonical_sha256":"b42b610cae5f03acb7d9695c469df23b184b219f5c20573623d0b63480fbd23c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b42b610cae5f03acb7d9695c469df23b184b219f5c20573623d0b63480fbd23c","first_computed_at":"2026-07-05T04:16:48.007443Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:16:48.007443Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FfFDf9SbE/0h2T1LFnnEelbJtn590N9BS4eTc38YBcMl0Uq0k0MKCK4aliYIvMLPpAXHFrwyl67rDCUdwF2eBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:16:48.007931Z","signed_message":"canonical_sha256_bytes"},"source_id":"2204.10216","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4bbc0045c8c95217752effabb5f4127b959399ea110ae7920df79f1adb91da4d","sha256:621ebcc3fd20f2864811a8990dc9efaa0e40d88e7a942a5d31bcf407049ea17a"],"state_sha256":"6bdd35fa375fc394eff2362d300a6b4ac94b133fc0e112858e31f5e5558bd612"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"O0xzpJysEYlmjejD5WS0UFib/BFnj12GPUrHDdjmNC6mH7OlnGRTFgbmTC8am+STnl7Vcmatsp/B1bRW9zN1CA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T02:57:48.945673Z","bundle_sha256":"cd5e0b76e053248fb43a31b40bea44c961795c32fa901a3b9e133b987f1dbfa3"}}