{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:CZG6ZOXYZCNWHX4SCZXTY7WXJK","short_pith_number":"pith:CZG6ZOXY","canonical_record":{"source":{"id":"2308.07286","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-14T17:17:21Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"cad8ccc1fca4123178163822b09b9fe97bc8f5fce4d22b3ec8dfa01d59159180","abstract_canon_sha256":"e4087a959c9406e114a2e0e4b71a8259fe4d71376250412bae4eee692655bcdd"},"schema_version":"1.0"},"canonical_sha256":"164decbaf8c89b63df92166f3c7ed74abb2a36ef0660adb11653b19f71e353e4","source":{"kind":"arxiv","id":"2308.07286","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.07286","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"arxiv_version","alias_value":"2308.07286v1","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.07286","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_12","alias_value":"CZG6ZOXYZCNW","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_16","alias_value":"CZG6ZOXYZCNWHX4S","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_8","alias_value":"CZG6ZOXY","created_at":"2026-07-05T06:40:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:CZG6ZOXYZCNWHX4SCZXTY7WXJK","target":"record","payload":{"canonical_record":{"source":{"id":"2308.07286","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-14T17:17:21Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"cad8ccc1fca4123178163822b09b9fe97bc8f5fce4d22b3ec8dfa01d59159180","abstract_canon_sha256":"e4087a959c9406e114a2e0e4b71a8259fe4d71376250412bae4eee692655bcdd"},"schema_version":"1.0"},"canonical_sha256":"164decbaf8c89b63df92166f3c7ed74abb2a36ef0660adb11653b19f71e353e4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:40:58.453557Z","signature_b64":"l/q13yvy4I4wb6Ypn1UBjN1h72NhLClgcoaTxSQNUo4EjAdbU0Y+STUNgoZxB2kIHkv6COFcrb65XhUsw3MDCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"164decbaf8c89b63df92166f3c7ed74abb2a36ef0660adb11653b19f71e353e4","last_reissued_at":"2026-07-05T06:40:58.452849Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:40:58.452849Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.07286","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:40:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3niJDGD8UBVTEYsRvS4VpUtBMUdc9IMkAclnrpTn/VVHTufZXcH4gxobh6AV1d/c2A87adFFcxm6Wplb9fXgCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:04:19.638881Z"},"content_sha256":"cf6e824eb013ed499a739daf627d667895851d71aff103915aada7ca58978397","schema_version":"1.0","event_id":"sha256:cf6e824eb013ed499a739daf627d667895851d71aff103915aada7ca58978397"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:CZG6ZOXYZCNWHX4SCZXTY7WXJK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Devil is in the Errors: Leveraging Large Language Models for Fine-grained Machine Translation Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Andr\\'e F. T. Martins, Ankush Garg, Daniel Deutsch, Graham Neubig, Jonathan H. Clark, Mara Finkelstein, Markus Freitag, Orhan Firat, Parker Riley, Patrick Fernandes","submitted_at":"2023-08-14T17:17:21Z","abstract_excerpt":"Automatic evaluation of machine translation (MT) is a critical tool driving the rapid iterative development of MT systems. While considerable progress has been made on estimating a single scalar quality score, current metrics lack the informativeness of more detailed schemes that annotate individual errors, such as Multidimensional Quality Metrics (MQM). In this paper, we help fill this gap by proposing AutoMQM, a prompting technique which leverages the reasoning and in-context learning capabilities of large language models (LLMs) and asks them to identify and categorize errors in translations"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.07286","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.07286/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:40:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"v0UY61pbd3lF4wwVHqjp60pUZfSUfNpnuOcrbVFe55vHRnLhbGO3MerCaWwx+jSJZQAk+yx7tzG2WQbMZ1xyBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:04:19.639196Z"},"content_sha256":"371e35302e754d692346b9ca6a8998fc8dd5c9448be8bcf5310bc67cb182b8e5","schema_version":"1.0","event_id":"sha256:371e35302e754d692346b9ca6a8998fc8dd5c9448be8bcf5310bc67cb182b8e5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/bundle.json","state_url":"https://pith.science/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T01:04:19Z","links":{"resolver":"https://pith.science/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK","bundle":"https://pith.science/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/bundle.json","state":"https://pith.science/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CZG6ZOXYZCNWHX4SCZXTY7WXJK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:CZG6ZOXYZCNWHX4SCZXTY7WXJK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e4087a959c9406e114a2e0e4b71a8259fe4d71376250412bae4eee692655bcdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-14T17:17:21Z","title_canon_sha256":"cad8ccc1fca4123178163822b09b9fe97bc8f5fce4d22b3ec8dfa01d59159180"},"schema_version":"1.0","source":{"id":"2308.07286","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.07286","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"arxiv_version","alias_value":"2308.07286v1","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.07286","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_12","alias_value":"CZG6ZOXYZCNW","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_16","alias_value":"CZG6ZOXYZCNWHX4S","created_at":"2026-07-05T06:40:58Z"},{"alias_kind":"pith_short_8","alias_value":"CZG6ZOXY","created_at":"2026-07-05T06:40:58Z"}],"graph_snapshots":[{"event_id":"sha256:371e35302e754d692346b9ca6a8998fc8dd5c9448be8bcf5310bc67cb182b8e5","target":"graph","created_at":"2026-07-05T06:40:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.07286/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic evaluation of machine translation (MT) is a critical tool driving the rapid iterative development of MT systems. While considerable progress has been made on estimating a single scalar quality score, current metrics lack the informativeness of more detailed schemes that annotate individual errors, such as Multidimensional Quality Metrics (MQM). In this paper, we help fill this gap by proposing AutoMQM, a prompting technique which leverages the reasoning and in-context learning capabilities of large language models (LLMs) and asks them to identify and categorize errors in translations","authors_text":"Andr\\'e F. T. Martins, Ankush Garg, Daniel Deutsch, Graham Neubig, Jonathan H. Clark, Mara Finkelstein, Markus Freitag, Orhan Firat, Parker Riley, Patrick Fernandes","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-14T17:17:21Z","title":"The Devil is in the Errors: Leveraging Large Language Models for Fine-grained Machine Translation Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.07286","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cf6e824eb013ed499a739daf627d667895851d71aff103915aada7ca58978397","target":"record","created_at":"2026-07-05T06:40:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e4087a959c9406e114a2e0e4b71a8259fe4d71376250412bae4eee692655bcdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-14T17:17:21Z","title_canon_sha256":"cad8ccc1fca4123178163822b09b9fe97bc8f5fce4d22b3ec8dfa01d59159180"},"schema_version":"1.0","source":{"id":"2308.07286","kind":"arxiv","version":1}},"canonical_sha256":"164decbaf8c89b63df92166f3c7ed74abb2a36ef0660adb11653b19f71e353e4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"164decbaf8c89b63df92166f3c7ed74abb2a36ef0660adb11653b19f71e353e4","first_computed_at":"2026-07-05T06:40:58.452849Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:40:58.452849Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"l/q13yvy4I4wb6Ypn1UBjN1h72NhLClgcoaTxSQNUo4EjAdbU0Y+STUNgoZxB2kIHkv6COFcrb65XhUsw3MDCA==","signature_status":"signed_v1","signed_at":"2026-07-05T06:40:58.453557Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.07286","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cf6e824eb013ed499a739daf627d667895851d71aff103915aada7ca58978397","sha256:371e35302e754d692346b9ca6a8998fc8dd5c9448be8bcf5310bc67cb182b8e5"],"state_sha256":"e891682e894008939c1fa1365d967bd8cea4c459aff1249d2236038b4bb2750e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XV4vP+98ImhMDSXYxCnTL1Rx4Zk03f3AaTBuQjPgltsxG0DBmQhw83se06adPC+93+gTrsszw+0lu7mlIDxSCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T01:04:19.642441Z","bundle_sha256":"b785c8e612fa148111e1f219cfd272139d3fada59cbffb828a0d203bf67cf766"}}