{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:F57ELTNXHWBEOI26YSB7S4TYBP","short_pith_number":"pith:F57ELTNX","schema_version":"1.0","canonical_sha256":"2f7e45cdb73d8247235ec483f972780bf1a6283bffeedd59390d3c172d10fedb","source":{"kind":"arxiv","id":"2305.14991","version":2},"attestation_state":"computed","paper":{"title":"MuLER: Detailed and Scalable Reference-based Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Gal Patel, Leshem Choshen, Omri Abend, Taelin Karidi","submitted_at":"2023-05-24T10:26:13Z","abstract_excerpt":"We propose a novel methodology (namely, MuLER) that transforms any reference-based evaluation metric for text generation, such as machine translation (MT) into a fine-grained analysis tool. Given a system and a metric, MuLER quantifies how much the chosen metric penalizes specific error types (e.g., errors in translating names of locations). MuLER thus enables a detailed error analysis which can lead to targeted improvement efforts for specific phenomena. We perform experiments in both synthetic and naturalistic settings to support MuLER's validity and showcase its usability in MT evaluation, "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.14991","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T10:26:13Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bf96cfc2e883995911ac4f1726901ed86ce0c801e3a57eb27c12866ba39097a6","abstract_canon_sha256":"3d9c4808f2a4973bbc79af427852b2322cdf83e04c2aa7f16224379eac7018ba"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:18:10.239663Z","signature_b64":"Y/UTnXKMVcdoDM/l5vDfg/zTTfhOYH4GJZRWiFHKIbviyOVS5cB7kQzciaxRqM4rf15IThbLB5ir9/UUDF1aCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f7e45cdb73d8247235ec483f972780bf1a6283bffeedd59390d3c172d10fedb","last_reissued_at":"2026-07-05T07:18:10.239174Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:18:10.239174Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MuLER: Detailed and Scalable Reference-based Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Gal Patel, Leshem Choshen, Omri Abend, Taelin Karidi","submitted_at":"2023-05-24T10:26:13Z","abstract_excerpt":"We propose a novel methodology (namely, MuLER) that transforms any reference-based evaluation metric for text generation, such as machine translation (MT) into a fine-grained analysis tool. Given a system and a metric, MuLER quantifies how much the chosen metric penalizes specific error types (e.g., errors in translating names of locations). MuLER thus enables a detailed error analysis which can lead to targeted improvement efforts for specific phenomena. We perform experiments in both synthetic and naturalistic settings to support MuLER's validity and showcase its usability in MT evaluation, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14991","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.14991/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.14991","created_at":"2026-07-05T07:18:10.239229+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.14991v2","created_at":"2026-07-05T07:18:10.239229+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14991","created_at":"2026-07-05T07:18:10.239229+00:00"},{"alias_kind":"pith_short_12","alias_value":"F57ELTNXHWBE","created_at":"2026-07-05T07:18:10.239229+00:00"},{"alias_kind":"pith_short_16","alias_value":"F57ELTNXHWBEOI26","created_at":"2026-07-05T07:18:10.239229+00:00"},{"alias_kind":"pith_short_8","alias_value":"F57ELTNX","created_at":"2026-07-05T07:18:10.239229+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP","json":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP.json","graph_json":"https://pith.science/api/pith-number/F57ELTNXHWBEOI26YSB7S4TYBP/graph.json","events_json":"https://pith.science/api/pith-number/F57ELTNXHWBEOI26YSB7S4TYBP/events.json","paper":"https://pith.science/paper/F57ELTNX"},"agent_actions":{"view_html":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP","download_json":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP.json","view_paper":"https://pith.science/paper/F57ELTNX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.14991&json=true","fetch_graph":"https://pith.science/api/pith-number/F57ELTNXHWBEOI26YSB7S4TYBP/graph.json","fetch_events":"https://pith.science/api/pith-number/F57ELTNXHWBEOI26YSB7S4TYBP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP/action/storage_attestation","attest_author":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP/action/author_attestation","sign_citation":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP/action/citation_signature","submit_replication":"https://pith.science/pith/F57ELTNXHWBEOI26YSB7S4TYBP/action/replication_record"}},"created_at":"2026-07-05T07:18:10.239229+00:00","updated_at":"2026-07-05T07:18:10.239229+00:00"}