{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:GKV5YRCQ45FZ6WYYGGC7SONFQF","short_pith_number":"pith:GKV5YRCQ","schema_version":"1.0","canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","source":{"kind":"arxiv","id":"2412.13989","version":1},"attestation_state":"computed","paper":{"title":"What makes a good metric? Evaluating automatic metrics for text-to-image consistency","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Adina Williams, Adriana Romero Soriano, Candace Ross, Melissa Hall","submitted_at":"2024-12-18T16:09:42Z","abstract_excerpt":"Language models are increasingly being incorporated as components in larger AI systems for various purposes, from prompt optimization to automatic evaluation. In this work, we analyze the construct validity of four recent, commonly used methods for measuring text-to-image consistency - CLIPScore, TIFA, VPEval, and DSG - which rely on language models and/or VQA models as components. We define construct validity for text-image consistency metrics as a set of desiderata that text-image consistency metrics should have, and find that no tested metric satisfies all of them. We find that metrics lack"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.13989","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","cross_cats_sorted":[],"title_canon_sha256":"86e6225ec550197281a69b688f6480d45dcc9ff1a084a02611252a7a69de7f87","abstract_canon_sha256":"e33a1f82e2cfbbc0130b0349828c719ddfbbac8b282a1e97441061e6d6991583"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:51:11.890184Z","signature_b64":"/Z10/XvXBxtQWwP7dgL2YGpJsd9XI502K3hOCPw542xO/iNMa/xXtC1y+RnvPrZ/XUqOEDUuxVx27LYlox4wDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","last_reissued_at":"2026-07-05T09:51:11.889711Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:51:11.889711Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"What makes a good metric? Evaluating automatic metrics for text-to-image consistency","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Adina Williams, Adriana Romero Soriano, Candace Ross, Melissa Hall","submitted_at":"2024-12-18T16:09:42Z","abstract_excerpt":"Language models are increasingly being incorporated as components in larger AI systems for various purposes, from prompt optimization to automatic evaluation. In this work, we analyze the construct validity of four recent, commonly used methods for measuring text-to-image consistency - CLIPScore, TIFA, VPEval, and DSG - which rely on language models and/or VQA models as components. We define construct validity for text-image consistency metrics as a set of desiderata that text-image consistency metrics should have, and find that no tested metric satisfies all of them. We find that metrics lack"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.13989","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.13989/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.13989","created_at":"2026-07-05T09:51:11.889769+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.13989v1","created_at":"2026-07-05T09:51:11.889769+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.13989","created_at":"2026-07-05T09:51:11.889769+00:00"},{"alias_kind":"pith_short_12","alias_value":"GKV5YRCQ45FZ","created_at":"2026-07-05T09:51:11.889769+00:00"},{"alias_kind":"pith_short_16","alias_value":"GKV5YRCQ45FZ6WYY","created_at":"2026-07-05T09:51:11.889769+00:00"},{"alias_kind":"pith_short_8","alias_value":"GKV5YRCQ","created_at":"2026-07-05T09:51:11.889769+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF","json":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF.json","graph_json":"https://pith.science/api/pith-number/GKV5YRCQ45FZ6WYYGGC7SONFQF/graph.json","events_json":"https://pith.science/api/pith-number/GKV5YRCQ45FZ6WYYGGC7SONFQF/events.json","paper":"https://pith.science/paper/GKV5YRCQ"},"agent_actions":{"view_html":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF","download_json":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF.json","view_paper":"https://pith.science/paper/GKV5YRCQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.13989&json=true","fetch_graph":"https://pith.science/api/pith-number/GKV5YRCQ45FZ6WYYGGC7SONFQF/graph.json","fetch_events":"https://pith.science/api/pith-number/GKV5YRCQ45FZ6WYYGGC7SONFQF/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/action/timestamp_anchor","attest_storage":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/action/storage_attestation","attest_author":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/action/author_attestation","sign_citation":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/action/citation_signature","submit_replication":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/action/replication_record"}},"created_at":"2026-07-05T09:51:11.889769+00:00","updated_at":"2026-07-05T09:51:11.889769+00:00"}