{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:S5EAHBHXLT2UDNX3FIWIHY37ZC","short_pith_number":"pith:S5EAHBHX","canonical_record":{"source":{"id":"2204.11574","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-25T11:41:50Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fcb9843c76d29d4b49bb787ac32d2911102a0996e954dddf0ee3eb0c833fd467","abstract_canon_sha256":"050a1dae29e49b42a29bd27ac3fb724e0b73bdb42a47ecb343be909f6433ada5"},"schema_version":"1.0"},"canonical_sha256":"97480384f75cf541b6fb2a2c83e37fc8bb1b19a0cc481fae9aae327a7529399a","source":{"kind":"arxiv","id":"2204.11574","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.11574","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"arxiv_version","alias_value":"2204.11574v1","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.11574","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_12","alias_value":"S5EAHBHXLT2U","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_16","alias_value":"S5EAHBHXLT2UDNX3","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_8","alias_value":"S5EAHBHX","created_at":"2026-07-05T04:17:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:S5EAHBHXLT2UDNX3FIWIHY37ZC","target":"record","payload":{"canonical_record":{"source":{"id":"2204.11574","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-25T11:41:50Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fcb9843c76d29d4b49bb787ac32d2911102a0996e954dddf0ee3eb0c833fd467","abstract_canon_sha256":"050a1dae29e49b42a29bd27ac3fb724e0b73bdb42a47ecb343be909f6433ada5"},"schema_version":"1.0"},"canonical_sha256":"97480384f75cf541b6fb2a2c83e37fc8bb1b19a0cc481fae9aae327a7529399a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:17:29.562170Z","signature_b64":"2SFkZDLUJMeqhaiiKbMiClTKviOCgDU2Ihz1C+7QFkBTfJCEeQVeD9CPrRFk1Esdr2ETWEpEiCO6Om9eHZcMDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"97480384f75cf541b6fb2a2c83e37fc8bb1b19a0cc481fae9aae327a7529399a","last_reissued_at":"2026-07-05T04:17:29.561706Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:17:29.561706Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2204.11574","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:17:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UySPx/YkndwJd0EoN8rEn0ArMxyzWfEwdjNb1bJNdgzIy3nd5oogPepDep266yZ2Q87rI8tPUpS30fdSxgPgDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:27:31.260169Z"},"content_sha256":"0236cbab5f6ada74fee832bf3b6b297963db44506cbe3b5114128ad6c470c12c","schema_version":"1.0","event_id":"sha256:0236cbab5f6ada74fee832bf3b6b297963db44506cbe3b5114128ad6c470c12c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:S5EAHBHXLT2UDNX3FIWIHY37ZC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A global analysis of metrics used for measuring performance in natural language processing","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Georg Dorffner, Kathrin Blagec, Matthias Samwald, Milad Moradi, Simon Ott","submitted_at":"2022-04-25T11:41:50Z","abstract_excerpt":"Measuring the performance of natural language processing models is challenging. Traditionally used metrics, such as BLEU and ROUGE, originally devised for machine translation and summarization, have been shown to suffer from low correlation with human judgment and a lack of transferability to other tasks and languages. In the past 15 years, a wide range of alternative metrics have been proposed. However, it is unclear to what extent this has had an impact on NLP benchmarking efforts. Here we provide the first large-scale cross-sectional analysis of metrics used for measuring performance in nat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.11574","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2204.11574/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:17:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"q9Bzop2wGTccfWxaAZGW81QoNSbqXQmtOM0HHyhkgdPu4cFvGlBKDuCKfnxr6esVUWfAvwX8bRVUUKQv1L+cCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:27:31.260726Z"},"content_sha256":"9a9ee3054cdfc0ed39d498dc7a6d1b9be6e56bf03e85d5985153568e8d5e324b","schema_version":"1.0","event_id":"sha256:9a9ee3054cdfc0ed39d498dc7a6d1b9be6e56bf03e85d5985153568e8d5e324b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/bundle.json","state_url":"https://pith.science/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T07:27:31Z","links":{"resolver":"https://pith.science/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC","bundle":"https://pith.science/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/bundle.json","state":"https://pith.science/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S5EAHBHXLT2UDNX3FIWIHY37ZC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:S5EAHBHXLT2UDNX3FIWIHY37ZC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"050a1dae29e49b42a29bd27ac3fb724e0b73bdb42a47ecb343be909f6433ada5","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-25T11:41:50Z","title_canon_sha256":"fcb9843c76d29d4b49bb787ac32d2911102a0996e954dddf0ee3eb0c833fd467"},"schema_version":"1.0","source":{"id":"2204.11574","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.11574","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"arxiv_version","alias_value":"2204.11574v1","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.11574","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_12","alias_value":"S5EAHBHXLT2U","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_16","alias_value":"S5EAHBHXLT2UDNX3","created_at":"2026-07-05T04:17:29Z"},{"alias_kind":"pith_short_8","alias_value":"S5EAHBHX","created_at":"2026-07-05T04:17:29Z"}],"graph_snapshots":[{"event_id":"sha256:9a9ee3054cdfc0ed39d498dc7a6d1b9be6e56bf03e85d5985153568e8d5e324b","target":"graph","created_at":"2026-07-05T04:17:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2204.11574/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Measuring the performance of natural language processing models is challenging. Traditionally used metrics, such as BLEU and ROUGE, originally devised for machine translation and summarization, have been shown to suffer from low correlation with human judgment and a lack of transferability to other tasks and languages. In the past 15 years, a wide range of alternative metrics have been proposed. However, it is unclear to what extent this has had an impact on NLP benchmarking efforts. Here we provide the first large-scale cross-sectional analysis of metrics used for measuring performance in nat","authors_text":"Georg Dorffner, Kathrin Blagec, Matthias Samwald, Milad Moradi, Simon Ott","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-25T11:41:50Z","title":"A global analysis of metrics used for measuring performance in natural language processing"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.11574","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0236cbab5f6ada74fee832bf3b6b297963db44506cbe3b5114128ad6c470c12c","target":"record","created_at":"2026-07-05T04:17:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"050a1dae29e49b42a29bd27ac3fb724e0b73bdb42a47ecb343be909f6433ada5","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-25T11:41:50Z","title_canon_sha256":"fcb9843c76d29d4b49bb787ac32d2911102a0996e954dddf0ee3eb0c833fd467"},"schema_version":"1.0","source":{"id":"2204.11574","kind":"arxiv","version":1}},"canonical_sha256":"97480384f75cf541b6fb2a2c83e37fc8bb1b19a0cc481fae9aae327a7529399a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"97480384f75cf541b6fb2a2c83e37fc8bb1b19a0cc481fae9aae327a7529399a","first_computed_at":"2026-07-05T04:17:29.561706Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:17:29.561706Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2SFkZDLUJMeqhaiiKbMiClTKviOCgDU2Ihz1C+7QFkBTfJCEeQVeD9CPrRFk1Esdr2ETWEpEiCO6Om9eHZcMDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:17:29.562170Z","signed_message":"canonical_sha256_bytes"},"source_id":"2204.11574","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0236cbab5f6ada74fee832bf3b6b297963db44506cbe3b5114128ad6c470c12c","sha256:9a9ee3054cdfc0ed39d498dc7a6d1b9be6e56bf03e85d5985153568e8d5e324b"],"state_sha256":"366eaa8c431fdf54958803b680fcf5673f9adf7ef4b360d91b2227c7905ac3a3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"O5Aq4ltS8dnLDELWTYBG7wyOaS5x3lEjSw6y7gi6RGfQAbEeZX8nyqGGA/jxv39ZCRhjXnAAXd3eXrrCkhwYBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T07:27:31.266087Z","bundle_sha256":"7f027e78d1815d6cbacb746885b0bcf8abb8e3df7196b198f1550f39c902af8e"}}