{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:ABOXN5VNMMW3FGL7PLZ3OMZQEA","short_pith_number":"pith:ABOXN5VN","canonical_record":{"source":{"id":"2210.13746","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-25T03:25:44Z","cross_cats_sorted":[],"title_canon_sha256":"80a971f1590325e3528e00d770a1e8e74bc65751b0619d9cfd2bf36fdcb4b230","abstract_canon_sha256":"7d9a2c7085483e4e3fd38ded42889eea2c945f17f216d818ba893467893086c4"},"schema_version":"1.0"},"canonical_sha256":"005d76f6ad632db2997f7af3b7333020361f2bd6086f3e6fba0efe5d1aae55e5","source":{"kind":"arxiv","id":"2210.13746","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.13746","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"arxiv_version","alias_value":"2210.13746v1","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.13746","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_12","alias_value":"ABOXN5VNMMW3","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_16","alias_value":"ABOXN5VNMMW3FGL7","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_8","alias_value":"ABOXN5VN","created_at":"2026-07-05T05:10:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:ABOXN5VNMMW3FGL7PLZ3OMZQEA","target":"record","payload":{"canonical_record":{"source":{"id":"2210.13746","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-25T03:25:44Z","cross_cats_sorted":[],"title_canon_sha256":"80a971f1590325e3528e00d770a1e8e74bc65751b0619d9cfd2bf36fdcb4b230","abstract_canon_sha256":"7d9a2c7085483e4e3fd38ded42889eea2c945f17f216d818ba893467893086c4"},"schema_version":"1.0"},"canonical_sha256":"005d76f6ad632db2997f7af3b7333020361f2bd6086f3e6fba0efe5d1aae55e5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:10:21.945700Z","signature_b64":"3gKgy792L8u8Q8QACowGV6AJHLFVRO6PY14Fcijw3xGdYvij3KogsPlODSepVm1b4ELlpf5gN0YRvRo1zhWbCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"005d76f6ad632db2997f7af3b7333020361f2bd6086f3e6fba0efe5d1aae55e5","last_reissued_at":"2026-07-05T05:10:21.945144Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:10:21.945144Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.13746","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CF1V6Pydgf5GVB04gWmI8TfA/ZIFi8ZQ+Bh4ETgYVb1kn8olbUJKklnIm/0RUQVEp6nO2+gcgZOzKod5ueSHAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:20:29.100073Z"},"content_sha256":"74ea186503d8c3078cd92bff32f0df53ac86cb6e79131c0338360957be25149e","schema_version":"1.0","event_id":"sha256:74ea186503d8c3078cd92bff32f0df53ac86cb6e79131c0338360957be25149e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:ABOXN5VNMMW3FGL7PLZ3OMZQEA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"DEMETR: Diagnosing Evaluation Metrics for Translation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ankita Gupta, Katherine Thai, Marzena Karpinska, Mohit Iyyer, Nishant Raj, Yixiao Song","submitted_at":"2022-10-25T03:25:44Z","abstract_excerpt":"While machine translation evaluation metrics based on string overlap (e.g., BLEU) have their limitations, their computations are transparent: the BLEU score assigned to a particular candidate translation can be traced back to the presence or absence of certain words. The operations of newer learned metrics (e.g., BLEURT, COMET), which leverage pretrained language models to achieve higher correlations with human quality judgments than BLEU, are opaque in comparison. In this paper, we shed light on the behavior of these learned metrics by creating DEMETR, a diagnostic dataset with 31K English ex"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.13746","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.13746/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JB7kKZExMClndMYRIXiA4UW7miPuYruM6G61G++uWxZ/hqSnlX7Izj0PcVtpAwPnMCdwj6enkDIQ9KNYT9ElBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:20:29.100612Z"},"content_sha256":"638063d0c6c10f62f1b113f39927e25c9396fa38340c6df9f2b2fbd908017579","schema_version":"1.0","event_id":"sha256:638063d0c6c10f62f1b113f39927e25c9396fa38340c6df9f2b2fbd908017579"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/bundle.json","state_url":"https://pith.science/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T07:20:29Z","links":{"resolver":"https://pith.science/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA","bundle":"https://pith.science/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/bundle.json","state":"https://pith.science/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ABOXN5VNMMW3FGL7PLZ3OMZQEA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:ABOXN5VNMMW3FGL7PLZ3OMZQEA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7d9a2c7085483e4e3fd38ded42889eea2c945f17f216d818ba893467893086c4","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-25T03:25:44Z","title_canon_sha256":"80a971f1590325e3528e00d770a1e8e74bc65751b0619d9cfd2bf36fdcb4b230"},"schema_version":"1.0","source":{"id":"2210.13746","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.13746","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"arxiv_version","alias_value":"2210.13746v1","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.13746","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_12","alias_value":"ABOXN5VNMMW3","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_16","alias_value":"ABOXN5VNMMW3FGL7","created_at":"2026-07-05T05:10:21Z"},{"alias_kind":"pith_short_8","alias_value":"ABOXN5VN","created_at":"2026-07-05T05:10:21Z"}],"graph_snapshots":[{"event_id":"sha256:638063d0c6c10f62f1b113f39927e25c9396fa38340c6df9f2b2fbd908017579","target":"graph","created_at":"2026-07-05T05:10:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.13746/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While machine translation evaluation metrics based on string overlap (e.g., BLEU) have their limitations, their computations are transparent: the BLEU score assigned to a particular candidate translation can be traced back to the presence or absence of certain words. The operations of newer learned metrics (e.g., BLEURT, COMET), which leverage pretrained language models to achieve higher correlations with human quality judgments than BLEU, are opaque in comparison. In this paper, we shed light on the behavior of these learned metrics by creating DEMETR, a diagnostic dataset with 31K English ex","authors_text":"Ankita Gupta, Katherine Thai, Marzena Karpinska, Mohit Iyyer, Nishant Raj, Yixiao Song","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-25T03:25:44Z","title":"DEMETR: Diagnosing Evaluation Metrics for Translation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.13746","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:74ea186503d8c3078cd92bff32f0df53ac86cb6e79131c0338360957be25149e","target":"record","created_at":"2026-07-05T05:10:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7d9a2c7085483e4e3fd38ded42889eea2c945f17f216d818ba893467893086c4","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-25T03:25:44Z","title_canon_sha256":"80a971f1590325e3528e00d770a1e8e74bc65751b0619d9cfd2bf36fdcb4b230"},"schema_version":"1.0","source":{"id":"2210.13746","kind":"arxiv","version":1}},"canonical_sha256":"005d76f6ad632db2997f7af3b7333020361f2bd6086f3e6fba0efe5d1aae55e5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"005d76f6ad632db2997f7af3b7333020361f2bd6086f3e6fba0efe5d1aae55e5","first_computed_at":"2026-07-05T05:10:21.945144Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:10:21.945144Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3gKgy792L8u8Q8QACowGV6AJHLFVRO6PY14Fcijw3xGdYvij3KogsPlODSepVm1b4ELlpf5gN0YRvRo1zhWbCA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:10:21.945700Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.13746","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:74ea186503d8c3078cd92bff32f0df53ac86cb6e79131c0338360957be25149e","sha256:638063d0c6c10f62f1b113f39927e25c9396fa38340c6df9f2b2fbd908017579"],"state_sha256":"aba06512c8dc589584f8173f0662f2cc1c2f1ab4a3bf1ad0242838b247e53f95"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qk3OWsOqBjKPuRItLI0AdsFnjhdNm+deYp/i7tg4sxZv9d0MojEU5VAnvnmaqMRd+1WV+u7NgNJNNRIti5LBCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T07:20:29.105038Z","bundle_sha256":"deb51ac3e42ee06ad89d283e803c2b055aa4a7554067613297d94b9923751b45"}}