{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:KZQLMM5VOHIJML3GPC4DNKRAAV","short_pith_number":"pith:KZQLMM5V","canonical_record":{"source":{"id":"2010.13588","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-26T13:57:20Z","cross_cats_sorted":[],"title_canon_sha256":"f6f80d74439ce98eee4190eafdfa360e74cd819fbf71502449f51393c3992045","abstract_canon_sha256":"5a2dc16f5a871c7d3fd780eb69fd9f56705c4c9522f3c55e5d40737452afe46a"},"schema_version":"1.0"},"canonical_sha256":"5660b633b571d0962f6678b836aa20057cbde6f626650e976795b0e7d36a8c08","source":{"kind":"arxiv","id":"2010.13588","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.13588","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"arxiv_version","alias_value":"2010.13588v1","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.13588","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_12","alias_value":"KZQLMM5VOHIJ","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_16","alias_value":"KZQLMM5VOHIJML3G","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_8","alias_value":"KZQLMM5V","created_at":"2026-07-05T01:46:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:KZQLMM5VOHIJML3GPC4DNKRAAV","target":"record","payload":{"canonical_record":{"source":{"id":"2010.13588","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-26T13:57:20Z","cross_cats_sorted":[],"title_canon_sha256":"f6f80d74439ce98eee4190eafdfa360e74cd819fbf71502449f51393c3992045","abstract_canon_sha256":"5a2dc16f5a871c7d3fd780eb69fd9f56705c4c9522f3c55e5d40737452afe46a"},"schema_version":"1.0"},"canonical_sha256":"5660b633b571d0962f6678b836aa20057cbde6f626650e976795b0e7d36a8c08","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:46:07.653276Z","signature_b64":"7Ek2zlXaZXQTYOuFai1yWBHXqsEe4X/Tv/NYi7N7QJxYo/Af0/I58OlSkIUHgK2Rx1Vy9d+pmviRjztVIV6hAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5660b633b571d0962f6678b836aa20057cbde6f626650e976795b0e7d36a8c08","last_reissued_at":"2026-07-05T01:46:07.652806Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:46:07.652806Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2010.13588","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Yzl8OzZeVggbR7TKtzwcVjbjV6FKh7L0kyXoUTUqN0Qh0qgfVVzefbjuxA0825WgeeyrSWn2sjttEJ9mrPsYBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-17T06:37:10.867462Z"},"content_sha256":"26841345f3e5d838a20f8901686bb82bd5e74525ea25788a8bf069eeb5049054","schema_version":"1.0","event_id":"sha256:26841345f3e5d838a20f8901686bb82bd5e74525ea25788a8bf069eeb5049054"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:KZQLMM5VOHIJML3GPC4DNKRAAV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Curious Case of Language Generation Evaluation Metrics: A Cautionary Tale","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Lucia Specia, Ozan Caglayan, Pranava Madhyastha","submitted_at":"2020-10-26T13:57:20Z","abstract_excerpt":"Automatic evaluation of language generation systems is a well-studied problem in Natural Language Processing. While novel metrics are proposed every year, a few popular metrics remain as the de facto metrics to evaluate tasks such as image captioning and machine translation, despite their known limitations. This is partly due to ease of use, and partly because researchers expect to see them and know how to interpret them. In this paper, we urge the community for more careful consideration of how they automatically evaluate their models by demonstrating important failure cases on multiple datas"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.13588","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.13588/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+8Skdl5U/BVtB9orGhEb0ZCvA129dKFK9Gg7Z63h8joBesG9sKJ9LZuLTW/NAAfLDtmNcHi6Fo3Ns7NZTBW8Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-17T06:37:10.867827Z"},"content_sha256":"7c2f2a37888f6570e2b603d3880a5948467c1397c8da6464dd4e5ad5a5d127df","schema_version":"1.0","event_id":"sha256:7c2f2a37888f6570e2b603d3880a5948467c1397c8da6464dd4e5ad5a5d127df"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/bundle.json","state_url":"https://pith.science/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-17T06:37:10Z","links":{"resolver":"https://pith.science/pith/KZQLMM5VOHIJML3GPC4DNKRAAV","bundle":"https://pith.science/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/bundle.json","state":"https://pith.science/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KZQLMM5VOHIJML3GPC4DNKRAAV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:KZQLMM5VOHIJML3GPC4DNKRAAV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5a2dc16f5a871c7d3fd780eb69fd9f56705c4c9522f3c55e5d40737452afe46a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-26T13:57:20Z","title_canon_sha256":"f6f80d74439ce98eee4190eafdfa360e74cd819fbf71502449f51393c3992045"},"schema_version":"1.0","source":{"id":"2010.13588","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.13588","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"arxiv_version","alias_value":"2010.13588v1","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.13588","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_12","alias_value":"KZQLMM5VOHIJ","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_16","alias_value":"KZQLMM5VOHIJML3G","created_at":"2026-07-05T01:46:07Z"},{"alias_kind":"pith_short_8","alias_value":"KZQLMM5V","created_at":"2026-07-05T01:46:07Z"}],"graph_snapshots":[{"event_id":"sha256:7c2f2a37888f6570e2b603d3880a5948467c1397c8da6464dd4e5ad5a5d127df","target":"graph","created_at":"2026-07-05T01:46:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2010.13588/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic evaluation of language generation systems is a well-studied problem in Natural Language Processing. While novel metrics are proposed every year, a few popular metrics remain as the de facto metrics to evaluate tasks such as image captioning and machine translation, despite their known limitations. This is partly due to ease of use, and partly because researchers expect to see them and know how to interpret them. In this paper, we urge the community for more careful consideration of how they automatically evaluate their models by demonstrating important failure cases on multiple datas","authors_text":"Lucia Specia, Ozan Caglayan, Pranava Madhyastha","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-26T13:57:20Z","title":"Curious Case of Language Generation Evaluation Metrics: A Cautionary Tale"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.13588","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:26841345f3e5d838a20f8901686bb82bd5e74525ea25788a8bf069eeb5049054","target":"record","created_at":"2026-07-05T01:46:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5a2dc16f5a871c7d3fd780eb69fd9f56705c4c9522f3c55e5d40737452afe46a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-26T13:57:20Z","title_canon_sha256":"f6f80d74439ce98eee4190eafdfa360e74cd819fbf71502449f51393c3992045"},"schema_version":"1.0","source":{"id":"2010.13588","kind":"arxiv","version":1}},"canonical_sha256":"5660b633b571d0962f6678b836aa20057cbde6f626650e976795b0e7d36a8c08","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5660b633b571d0962f6678b836aa20057cbde6f626650e976795b0e7d36a8c08","first_computed_at":"2026-07-05T01:46:07.652806Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:46:07.652806Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"7Ek2zlXaZXQTYOuFai1yWBHXqsEe4X/Tv/NYi7N7QJxYo/Af0/I58OlSkIUHgK2Rx1Vy9d+pmviRjztVIV6hAA==","signature_status":"signed_v1","signed_at":"2026-07-05T01:46:07.653276Z","signed_message":"canonical_sha256_bytes"},"source_id":"2010.13588","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:26841345f3e5d838a20f8901686bb82bd5e74525ea25788a8bf069eeb5049054","sha256:7c2f2a37888f6570e2b603d3880a5948467c1397c8da6464dd4e5ad5a5d127df"],"state_sha256":"a774dfeb80b898236f2f59da38627c77c7d6981453497dda6a94d455c2ecab35"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FWMi3LRh8S1IB+rRo8ldXvY5OBqgngZHYXROx/16vczddAwmo/L5RQ4SdmjWJJYvHZDtTz1wFPtE/k3hi5qnCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-17T06:37:10.869905Z","bundle_sha256":"454296d85f15d451f932e003b0946166345eef0f637d98692caadeb02e96961d"}}