{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:ET5KI6DWE2ZONQB6JLKGJVDP2X","short_pith_number":"pith:ET5KI6DW","canonical_record":{"source":{"id":"2401.16313","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T17:17:42Z","cross_cats_sorted":[],"title_canon_sha256":"516311241a2721cbc510d17ba5bf449479529134dae73ad576719f3de423b768","abstract_canon_sha256":"9d667326bd1e8cab70e741000a537b00ea95f3cf00da57d20a739550b20991ea"},"schema_version":"1.0"},"canonical_sha256":"24faa4787626b2e6c03e4ad464d46fd5f4e1959d1f01fe03e20ae3863b60017d","source":{"kind":"arxiv","id":"2401.16313","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16313","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16313v1","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16313","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_12","alias_value":"ET5KI6DWE2ZO","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_16","alias_value":"ET5KI6DWE2ZONQB6","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_8","alias_value":"ET5KI6DW","created_at":"2026-07-05T07:38:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:ET5KI6DWE2ZONQB6JLKGJVDP2X","target":"record","payload":{"canonical_record":{"source":{"id":"2401.16313","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T17:17:42Z","cross_cats_sorted":[],"title_canon_sha256":"516311241a2721cbc510d17ba5bf449479529134dae73ad576719f3de423b768","abstract_canon_sha256":"9d667326bd1e8cab70e741000a537b00ea95f3cf00da57d20a739550b20991ea"},"schema_version":"1.0"},"canonical_sha256":"24faa4787626b2e6c03e4ad464d46fd5f4e1959d1f01fe03e20ae3863b60017d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:38:47.408251Z","signature_b64":"saym0wHvKc2KZnPWmy1iJQfa6IEe884Axt8svKxsE7vanoKhqGfUsO//VJzO3peRt3HN2qiEbmgGL5z8NdBLBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"24faa4787626b2e6c03e4ad464d46fd5f4e1959d1f01fe03e20ae3863b60017d","last_reissued_at":"2026-07-05T07:38:47.407766Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:38:47.407766Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.16313","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kmXuKESYZgVa4/CI4y9P5GszQVbyVe0Nit4XuWtBtcHHjpZi48cb8fPGOqO44nbiRjX/Cyfcir9tkeXMG8/+Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T06:49:40.009719Z"},"content_sha256":"aef67d60ffc270f6892d3d771f27ee8d8aae80610e044a46d4607869dfdcc1b9","schema_version":"1.0","event_id":"sha256:aef67d60ffc270f6892d3d771f27ee8d8aae80610e044a46d4607869dfdcc1b9"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:ET5KI6DWE2ZONQB6JLKGJVDP2X","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Machine Translation Meta Evaluation through Translation Accuracy Challenge Sets","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexandra Birch, Arnisa Fazla, Chantal Amrhein, Liane Guillou, Mark Steedman, Nikita Moghe, Rico Sennrich, Tom Kocmi","submitted_at":"2024-01-29T17:17:42Z","abstract_excerpt":"Recent machine translation (MT) metrics calibrate their effectiveness by correlating with human judgement but without any insights about their behaviour across different error types. Challenge sets are used to probe specific dimensions of metric behaviour but there are very few such datasets and they either focus on a limited number of phenomena or a limited number of language pairs. We introduce ACES, a contrastive challenge set spanning 146 language pairs, aimed at discovering whether metrics can identify 68 translation accuracy errors. These phenomena range from simple alterations at the wo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16313","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.16313/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lu0gyu7ymXSuJ2J2t5XCwrEUCmR7K/3UYlpgUGNM465oD/GofyIVmWbTL+x/HgawT4rkaO1paZRDPQZOfL6ZDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T06:49:40.010276Z"},"content_sha256":"de827c10251febc4c01cd7eba45599b3d8b31b5aa653668e5cae13b98eedf2e9","schema_version":"1.0","event_id":"sha256:de827c10251febc4c01cd7eba45599b3d8b31b5aa653668e5cae13b98eedf2e9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/bundle.json","state_url":"https://pith.science/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T06:49:40Z","links":{"resolver":"https://pith.science/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X","bundle":"https://pith.science/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/bundle.json","state":"https://pith.science/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ET5KI6DWE2ZONQB6JLKGJVDP2X/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ET5KI6DWE2ZONQB6JLKGJVDP2X","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9d667326bd1e8cab70e741000a537b00ea95f3cf00da57d20a739550b20991ea","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T17:17:42Z","title_canon_sha256":"516311241a2721cbc510d17ba5bf449479529134dae73ad576719f3de423b768"},"schema_version":"1.0","source":{"id":"2401.16313","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16313","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16313v1","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16313","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_12","alias_value":"ET5KI6DWE2ZO","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_16","alias_value":"ET5KI6DWE2ZONQB6","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_8","alias_value":"ET5KI6DW","created_at":"2026-07-05T07:38:47Z"}],"graph_snapshots":[{"event_id":"sha256:de827c10251febc4c01cd7eba45599b3d8b31b5aa653668e5cae13b98eedf2e9","target":"graph","created_at":"2026-07-05T07:38:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.16313/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent machine translation (MT) metrics calibrate their effectiveness by correlating with human judgement but without any insights about their behaviour across different error types. Challenge sets are used to probe specific dimensions of metric behaviour but there are very few such datasets and they either focus on a limited number of phenomena or a limited number of language pairs. We introduce ACES, a contrastive challenge set spanning 146 language pairs, aimed at discovering whether metrics can identify 68 translation accuracy errors. These phenomena range from simple alterations at the wo","authors_text":"Alexandra Birch, Arnisa Fazla, Chantal Amrhein, Liane Guillou, Mark Steedman, Nikita Moghe, Rico Sennrich, Tom Kocmi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T17:17:42Z","title":"Machine Translation Meta Evaluation through Translation Accuracy Challenge Sets"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16313","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:aef67d60ffc270f6892d3d771f27ee8d8aae80610e044a46d4607869dfdcc1b9","target":"record","created_at":"2026-07-05T07:38:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9d667326bd1e8cab70e741000a537b00ea95f3cf00da57d20a739550b20991ea","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-29T17:17:42Z","title_canon_sha256":"516311241a2721cbc510d17ba5bf449479529134dae73ad576719f3de423b768"},"schema_version":"1.0","source":{"id":"2401.16313","kind":"arxiv","version":1}},"canonical_sha256":"24faa4787626b2e6c03e4ad464d46fd5f4e1959d1f01fe03e20ae3863b60017d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"24faa4787626b2e6c03e4ad464d46fd5f4e1959d1f01fe03e20ae3863b60017d","first_computed_at":"2026-07-05T07:38:47.407766Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:38:47.407766Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"saym0wHvKc2KZnPWmy1iJQfa6IEe884Axt8svKxsE7vanoKhqGfUsO//VJzO3peRt3HN2qiEbmgGL5z8NdBLBw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:38:47.408251Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.16313","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:aef67d60ffc270f6892d3d771f27ee8d8aae80610e044a46d4607869dfdcc1b9","sha256:de827c10251febc4c01cd7eba45599b3d8b31b5aa653668e5cae13b98eedf2e9"],"state_sha256":"310d8e8fc14289556799e3f9e06096e8429fe4fe5216e328879101a2a5a6be9d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"w1CqHEPoBuXSJyGfGBKGb+lLfmGetTPc14w+kuaAHIId1EeMjdKR168rDp5bLx7Jmb/psifKPII1g+9dno0tDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T06:49:40.013987Z","bundle_sha256":"0702511a78ed87a326c7e857b2c6fa673811be56253ee582a8faf3f2a49b0d70"}}