{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:WIRDGXYFMRXBC7AUP5IHJML6N7","short_pith_number":"pith:WIRDGXYF","canonical_record":{"source":{"id":"2104.14478","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-29T16:42:09Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"bf998acc7245edf83bf1287c0824a947aa0862ae06c38f6f39fe379f4f1fb9c2","abstract_canon_sha256":"f97107efc8f625870668fd552b34712bf1e6652a83e3010d6000a7b5fdf4fafc"},"schema_version":"1.0"},"canonical_sha256":"b222335f05646e117c147f5074b17e6fdc483d883159ffd63975dadcb2437c09","source":{"kind":"arxiv","id":"2104.14478","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.14478","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"arxiv_version","alias_value":"2104.14478v1","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.14478","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_12","alias_value":"WIRDGXYFMRXB","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_16","alias_value":"WIRDGXYFMRXBC7AU","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_8","alias_value":"WIRDGXYF","created_at":"2026-07-05T04:17:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:WIRDGXYFMRXBC7AUP5IHJML6N7","target":"record","payload":{"canonical_record":{"source":{"id":"2104.14478","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-29T16:42:09Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"bf998acc7245edf83bf1287c0824a947aa0862ae06c38f6f39fe379f4f1fb9c2","abstract_canon_sha256":"f97107efc8f625870668fd552b34712bf1e6652a83e3010d6000a7b5fdf4fafc"},"schema_version":"1.0"},"canonical_sha256":"b222335f05646e117c147f5074b17e6fdc483d883159ffd63975dadcb2437c09","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:17:35.146782Z","signature_b64":"32wGY3sVzLRDj0uz8BJUrlJqDibtzSYu1t21vfwDjNXpNLOOychpW0zrUYfEsYgcvsY22IHXCerxQeRnL9ENAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b222335f05646e117c147f5074b17e6fdc483d883159ffd63975dadcb2437c09","last_reissued_at":"2026-07-05T04:17:35.146289Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:17:35.146289Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.14478","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:17:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IltUPLQRBgqD2K8NBhhkL+TcXpZoErcWf5BUjSAC2BrSkar11DV9VlxUcQD2aUUtpP/UO9b/zKkcQ/1ZyapqDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T06:08:37.478145Z"},"content_sha256":"c995deee6cf84e5d02e01002dab72bc0d0cbfb6097e5e2353f63344c3f2a305a","schema_version":"1.0","event_id":"sha256:c995deee6cf84e5d02e01002dab72bc0d0cbfb6097e5e2353f63344c3f2a305a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:WIRDGXYFMRXBC7AUP5IHJML6N7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Experts, Errors, and Context: A Large-Scale Study of Human Evaluation for Machine Translation","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"David Grangier, George Foster, Markus Freitag, Qijun Tan, Viresh Ratnakar, Wolfgang Macherey","submitted_at":"2021-04-29T16:42:09Z","abstract_excerpt":"Human evaluation of modern high-quality machine translation systems is a difficult problem, and there is increasing evidence that inadequate evaluation procedures can lead to erroneous conclusions. While there has been considerable research on human evaluation, the field still lacks a commonly-accepted standard procedure. As a step toward this goal, we propose an evaluation methodology grounded in explicit error analysis, based on the Multidimensional Quality Metrics (MQM) framework. We carry out the largest MQM research study to date, scoring the outputs of top systems from the WMT 2020 share"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.14478","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.14478/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:17:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"V+kR9ULItU6yTglIbbFAwZ0C+ufDKg6EHsQcBmmUHdkaV9KSaoENdwjXVMmVwdC/vEKr3xeSnbq5uX1blGQsDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T06:08:37.479080Z"},"content_sha256":"09c1498f1d527bc6ab4611ddd5dae5e2629ac426ef19e3bb0014a988a430f512","schema_version":"1.0","event_id":"sha256:09c1498f1d527bc6ab4611ddd5dae5e2629ac426ef19e3bb0014a988a430f512"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/bundle.json","state_url":"https://pith.science/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T06:08:37Z","links":{"resolver":"https://pith.science/pith/WIRDGXYFMRXBC7AUP5IHJML6N7","bundle":"https://pith.science/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/bundle.json","state":"https://pith.science/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WIRDGXYFMRXBC7AUP5IHJML6N7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:WIRDGXYFMRXBC7AUP5IHJML6N7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f97107efc8f625870668fd552b34712bf1e6652a83e3010d6000a7b5fdf4fafc","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-29T16:42:09Z","title_canon_sha256":"bf998acc7245edf83bf1287c0824a947aa0862ae06c38f6f39fe379f4f1fb9c2"},"schema_version":"1.0","source":{"id":"2104.14478","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.14478","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"arxiv_version","alias_value":"2104.14478v1","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.14478","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_12","alias_value":"WIRDGXYFMRXB","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_16","alias_value":"WIRDGXYFMRXBC7AU","created_at":"2026-07-05T04:17:35Z"},{"alias_kind":"pith_short_8","alias_value":"WIRDGXYF","created_at":"2026-07-05T04:17:35Z"}],"graph_snapshots":[{"event_id":"sha256:09c1498f1d527bc6ab4611ddd5dae5e2629ac426ef19e3bb0014a988a430f512","target":"graph","created_at":"2026-07-05T04:17:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.14478/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Human evaluation of modern high-quality machine translation systems is a difficult problem, and there is increasing evidence that inadequate evaluation procedures can lead to erroneous conclusions. While there has been considerable research on human evaluation, the field still lacks a commonly-accepted standard procedure. As a step toward this goal, we propose an evaluation methodology grounded in explicit error analysis, based on the Multidimensional Quality Metrics (MQM) framework. We carry out the largest MQM research study to date, scoring the outputs of top systems from the WMT 2020 share","authors_text":"David Grangier, George Foster, Markus Freitag, Qijun Tan, Viresh Ratnakar, Wolfgang Macherey","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-29T16:42:09Z","title":"Experts, Errors, and Context: A Large-Scale Study of Human Evaluation for Machine Translation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.14478","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c995deee6cf84e5d02e01002dab72bc0d0cbfb6097e5e2353f63344c3f2a305a","target":"record","created_at":"2026-07-05T04:17:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f97107efc8f625870668fd552b34712bf1e6652a83e3010d6000a7b5fdf4fafc","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-29T16:42:09Z","title_canon_sha256":"bf998acc7245edf83bf1287c0824a947aa0862ae06c38f6f39fe379f4f1fb9c2"},"schema_version":"1.0","source":{"id":"2104.14478","kind":"arxiv","version":1}},"canonical_sha256":"b222335f05646e117c147f5074b17e6fdc483d883159ffd63975dadcb2437c09","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b222335f05646e117c147f5074b17e6fdc483d883159ffd63975dadcb2437c09","first_computed_at":"2026-07-05T04:17:35.146289Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:17:35.146289Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"32wGY3sVzLRDj0uz8BJUrlJqDibtzSYu1t21vfwDjNXpNLOOychpW0zrUYfEsYgcvsY22IHXCerxQeRnL9ENAw==","signature_status":"signed_v1","signed_at":"2026-07-05T04:17:35.146782Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.14478","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c995deee6cf84e5d02e01002dab72bc0d0cbfb6097e5e2353f63344c3f2a305a","sha256:09c1498f1d527bc6ab4611ddd5dae5e2629ac426ef19e3bb0014a988a430f512"],"state_sha256":"07b28d1643057390f42ad8de1ba83fe23de400f59415eb35c434ea04cbaef950"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bYfqDmTVwQIIOGHQ7v+fqhicqdAvnjvnOsKkS4QHbknVwTvu44qP7fkWBjyHj+KbGh7ntORU/c5MWc1A28yhCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T06:08:37.484424Z","bundle_sha256":"eef957c8054a8822d69d20231aa1119f3e9f6429e89dca1c1fb3aa20629d1fa0"}}