{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:U6BNUWMHMLSOEUBSJUVC2LDVEK","short_pith_number":"pith:U6BNUWMH","canonical_record":{"source":{"id":"2107.10821","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-07-22T17:22:22Z","cross_cats_sorted":[],"title_canon_sha256":"fd7ea46a09f8dc87b15484cb5c66556c1518c15643f1b0e0e82e4ee2c3ee9c69","abstract_canon_sha256":"68f8f8832579014e93f66eaa017de03d6913a39730033eddb57af7afcbdde83f"},"schema_version":"1.0"},"canonical_sha256":"a782da598762e4e250324d2a2d2c752291f8510fcef35c12a302c3cf6ec652f5","source":{"kind":"arxiv","id":"2107.10821","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.10821","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"arxiv_version","alias_value":"2107.10821v2","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.10821","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_12","alias_value":"U6BNUWMHMLSO","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_16","alias_value":"U6BNUWMHMLSOEUBS","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_8","alias_value":"U6BNUWMH","created_at":"2026-07-05T03:13:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:U6BNUWMHMLSOEUBSJUVC2LDVEK","target":"record","payload":{"canonical_record":{"source":{"id":"2107.10821","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-07-22T17:22:22Z","cross_cats_sorted":[],"title_canon_sha256":"fd7ea46a09f8dc87b15484cb5c66556c1518c15643f1b0e0e82e4ee2c3ee9c69","abstract_canon_sha256":"68f8f8832579014e93f66eaa017de03d6913a39730033eddb57af7afcbdde83f"},"schema_version":"1.0"},"canonical_sha256":"a782da598762e4e250324d2a2d2c752291f8510fcef35c12a302c3cf6ec652f5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:13:50.410150Z","signature_b64":"QaQeST/SbDB7JbWqOWJYbknIhR2WX4kZzi/m7mj6E8X3XV71oEuTViRUlPx2lN9X8jLr5L5Y/Un7SLB7artjBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a782da598762e4e250324d2a2d2c752291f8510fcef35c12a302c3cf6ec652f5","last_reissued_at":"2026-07-05T03:13:50.409758Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:13:50.409758Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.10821","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:13:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"F0AJtpqAI5FBDxrOp7if2pN9VjruHdRVBhOaTossBz0TKLBdXfQFx6AW7xmfDMIFNc1NSvkr+ys9BNgfTvUWAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T18:07:51.742579Z"},"content_sha256":"1dbd694920b7ffe89dbbddfc093a16ee4aead9253681cdf613c14932e3fd7604","schema_version":"1.0","event_id":"sha256:1dbd694920b7ffe89dbbddfc093a16ee4aead9253681cdf613c14932e3fd7604"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:U6BNUWMHMLSOEUBSJUVC2LDVEK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"To Ship or Not to Ship: An Extensive Evaluation of Automatic Metrics for Machine Translation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Arul Menezes, Christian Federmann, Hitokazu Matsushita, Marcin Junczys-Dowmunt, Roman Grundkiewicz, Tom Kocmi","submitted_at":"2021-07-22T17:22:22Z","abstract_excerpt":"Automatic metrics are commonly used as the exclusive tool for declaring the superiority of one machine translation system's quality over another. The community choice of automatic metric guides research directions and industrial developments by deciding which models are deemed better. Evaluating metrics correlations with sets of human judgements has been limited by the size of these sets. In this paper, we corroborate how reliable metrics are in contrast to human judgements on -- to the best of our knowledge -- the largest collection of judgements reported in the literature. Arguably, pairwise"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.10821","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.10821/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:13:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9j/OBED/Gh7DNtLIocDbUcSzPquhxQcojHw3f9NQexbUWbiJEM0dSAG9phOAVKipyANOrf6urtSnJMsDQClHBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T18:07:51.743532Z"},"content_sha256":"535fd5903571fc35190324c73bb5e6eab526d915a128a7c952dc0146a3e7ca10","schema_version":"1.0","event_id":"sha256:535fd5903571fc35190324c73bb5e6eab526d915a128a7c952dc0146a3e7ca10"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/bundle.json","state_url":"https://pith.science/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T18:07:51Z","links":{"resolver":"https://pith.science/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK","bundle":"https://pith.science/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/bundle.json","state":"https://pith.science/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/U6BNUWMHMLSOEUBSJUVC2LDVEK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:U6BNUWMHMLSOEUBSJUVC2LDVEK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"68f8f8832579014e93f66eaa017de03d6913a39730033eddb57af7afcbdde83f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-07-22T17:22:22Z","title_canon_sha256":"fd7ea46a09f8dc87b15484cb5c66556c1518c15643f1b0e0e82e4ee2c3ee9c69"},"schema_version":"1.0","source":{"id":"2107.10821","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.10821","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"arxiv_version","alias_value":"2107.10821v2","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.10821","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_12","alias_value":"U6BNUWMHMLSO","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_16","alias_value":"U6BNUWMHMLSOEUBS","created_at":"2026-07-05T03:13:50Z"},{"alias_kind":"pith_short_8","alias_value":"U6BNUWMH","created_at":"2026-07-05T03:13:50Z"}],"graph_snapshots":[{"event_id":"sha256:535fd5903571fc35190324c73bb5e6eab526d915a128a7c952dc0146a3e7ca10","target":"graph","created_at":"2026-07-05T03:13:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.10821/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic metrics are commonly used as the exclusive tool for declaring the superiority of one machine translation system's quality over another. The community choice of automatic metric guides research directions and industrial developments by deciding which models are deemed better. Evaluating metrics correlations with sets of human judgements has been limited by the size of these sets. In this paper, we corroborate how reliable metrics are in contrast to human judgements on -- to the best of our knowledge -- the largest collection of judgements reported in the literature. Arguably, pairwise","authors_text":"Arul Menezes, Christian Federmann, Hitokazu Matsushita, Marcin Junczys-Dowmunt, Roman Grundkiewicz, Tom Kocmi","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-07-22T17:22:22Z","title":"To Ship or Not to Ship: An Extensive Evaluation of Automatic Metrics for Machine Translation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.10821","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1dbd694920b7ffe89dbbddfc093a16ee4aead9253681cdf613c14932e3fd7604","target":"record","created_at":"2026-07-05T03:13:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"68f8f8832579014e93f66eaa017de03d6913a39730033eddb57af7afcbdde83f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-07-22T17:22:22Z","title_canon_sha256":"fd7ea46a09f8dc87b15484cb5c66556c1518c15643f1b0e0e82e4ee2c3ee9c69"},"schema_version":"1.0","source":{"id":"2107.10821","kind":"arxiv","version":2}},"canonical_sha256":"a782da598762e4e250324d2a2d2c752291f8510fcef35c12a302c3cf6ec652f5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a782da598762e4e250324d2a2d2c752291f8510fcef35c12a302c3cf6ec652f5","first_computed_at":"2026-07-05T03:13:50.409758Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:13:50.409758Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"QaQeST/SbDB7JbWqOWJYbknIhR2WX4kZzi/m7mj6E8X3XV71oEuTViRUlPx2lN9X8jLr5L5Y/Un7SLB7artjBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T03:13:50.410150Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.10821","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1dbd694920b7ffe89dbbddfc093a16ee4aead9253681cdf613c14932e3fd7604","sha256:535fd5903571fc35190324c73bb5e6eab526d915a128a7c952dc0146a3e7ca10"],"state_sha256":"fe38dfb17952834a52705d97454b03a9ba8d0bcca529c4939042b17ee477e4d7"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gngyi0iUADw34NoRvT6YAWVtvxYQnbFenVIynqml+B7rhpJHOTPLWR+Fm9DQoiYoFrteBVhHm3aGTtSRuL3pDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T18:07:51.748801Z","bundle_sha256":"4af72e6d6909c4cfeb07c1295dda546245a3e301354dea492d449bbecb916543"}}