{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GKV5YRCQ45FZ6WYYGGC7SONFQF","short_pith_number":"pith:GKV5YRCQ","canonical_record":{"source":{"id":"2412.13989","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","cross_cats_sorted":[],"title_canon_sha256":"86e6225ec550197281a69b688f6480d45dcc9ff1a084a02611252a7a69de7f87","abstract_canon_sha256":"e33a1f82e2cfbbc0130b0349828c719ddfbbac8b282a1e97441061e6d6991583"},"schema_version":"1.0"},"canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","source":{"kind":"arxiv","id":"2412.13989","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.13989","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"arxiv_version","alias_value":"2412.13989v1","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.13989","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_12","alias_value":"GKV5YRCQ45FZ","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_16","alias_value":"GKV5YRCQ45FZ6WYY","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_8","alias_value":"GKV5YRCQ","created_at":"2026-07-05T09:51:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GKV5YRCQ45FZ6WYYGGC7SONFQF","target":"record","payload":{"canonical_record":{"source":{"id":"2412.13989","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","cross_cats_sorted":[],"title_canon_sha256":"86e6225ec550197281a69b688f6480d45dcc9ff1a084a02611252a7a69de7f87","abstract_canon_sha256":"e33a1f82e2cfbbc0130b0349828c719ddfbbac8b282a1e97441061e6d6991583"},"schema_version":"1.0"},"canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:51:11.890184Z","signature_b64":"/Z10/XvXBxtQWwP7dgL2YGpJsd9XI502K3hOCPw542xO/iNMa/xXtC1y+RnvPrZ/XUqOEDUuxVx27LYlox4wDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","last_reissued_at":"2026-07-05T09:51:11.889711Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:51:11.889711Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.13989","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:51:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YEf21wY4E3LoqP4vtEtiD65Tqo+8ihJZ4IzFFwB3CDtOnjU7FkQq12WlW2/zPpNz1olWJmVWPLHxSSafwqB7Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T07:57:49.713401Z"},"content_sha256":"7abad14695e8d2e3d7fffaccb38db52898e51ff7b98286176652d7e4f56ea797","schema_version":"1.0","event_id":"sha256:7abad14695e8d2e3d7fffaccb38db52898e51ff7b98286176652d7e4f56ea797"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GKV5YRCQ45FZ6WYYGGC7SONFQF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"What makes a good metric? Evaluating automatic metrics for text-to-image consistency","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Adina Williams, Adriana Romero Soriano, Candace Ross, Melissa Hall","submitted_at":"2024-12-18T16:09:42Z","abstract_excerpt":"Language models are increasingly being incorporated as components in larger AI systems for various purposes, from prompt optimization to automatic evaluation. In this work, we analyze the construct validity of four recent, commonly used methods for measuring text-to-image consistency - CLIPScore, TIFA, VPEval, and DSG - which rely on language models and/or VQA models as components. We define construct validity for text-image consistency metrics as a set of desiderata that text-image consistency metrics should have, and find that no tested metric satisfies all of them. We find that metrics lack"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.13989","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.13989/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:51:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"r/+LLD71+dmyCX6TeOAGkRnugJt9DzxDzFPn3EePr2lLxa3lHrVtc1Qvglt2yQhAxh5uBYexEM9UhRwJHpd1BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T07:57:49.713776Z"},"content_sha256":"8134e4ee68ab989d46062ae6c26f4c686168ce7b4e50d592a864a3d9ce938cf1","schema_version":"1.0","event_id":"sha256:8134e4ee68ab989d46062ae6c26f4c686168ce7b4e50d592a864a3d9ce938cf1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/bundle.json","state_url":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T07:57:49Z","links":{"resolver":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF","bundle":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/bundle.json","state":"https://pith.science/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GKV5YRCQ45FZ6WYYGGC7SONFQF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GKV5YRCQ45FZ6WYYGGC7SONFQF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e33a1f82e2cfbbc0130b0349828c719ddfbbac8b282a1e97441061e6d6991583","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","title_canon_sha256":"86e6225ec550197281a69b688f6480d45dcc9ff1a084a02611252a7a69de7f87"},"schema_version":"1.0","source":{"id":"2412.13989","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.13989","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"arxiv_version","alias_value":"2412.13989v1","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.13989","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_12","alias_value":"GKV5YRCQ45FZ","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_16","alias_value":"GKV5YRCQ45FZ6WYY","created_at":"2026-07-05T09:51:11Z"},{"alias_kind":"pith_short_8","alias_value":"GKV5YRCQ","created_at":"2026-07-05T09:51:11Z"}],"graph_snapshots":[{"event_id":"sha256:8134e4ee68ab989d46062ae6c26f4c686168ce7b4e50d592a864a3d9ce938cf1","target":"graph","created_at":"2026-07-05T09:51:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.13989/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language models are increasingly being incorporated as components in larger AI systems for various purposes, from prompt optimization to automatic evaluation. In this work, we analyze the construct validity of four recent, commonly used methods for measuring text-to-image consistency - CLIPScore, TIFA, VPEval, and DSG - which rely on language models and/or VQA models as components. We define construct validity for text-image consistency metrics as a set of desiderata that text-image consistency metrics should have, and find that no tested metric satisfies all of them. We find that metrics lack","authors_text":"Adina Williams, Adriana Romero Soriano, Candace Ross, Melissa Hall","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","title":"What makes a good metric? Evaluating automatic metrics for text-to-image consistency"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.13989","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7abad14695e8d2e3d7fffaccb38db52898e51ff7b98286176652d7e4f56ea797","target":"record","created_at":"2026-07-05T09:51:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e33a1f82e2cfbbc0130b0349828c719ddfbbac8b282a1e97441061e6d6991583","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-18T16:09:42Z","title_canon_sha256":"86e6225ec550197281a69b688f6480d45dcc9ff1a084a02611252a7a69de7f87"},"schema_version":"1.0","source":{"id":"2412.13989","kind":"arxiv","version":1}},"canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"32abdc4450e74b9f5b183185f939a581552a22353a5a9fe5e6ad7cae06c7f838","first_computed_at":"2026-07-05T09:51:11.889711Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:51:11.889711Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/Z10/XvXBxtQWwP7dgL2YGpJsd9XI502K3hOCPw542xO/iNMa/xXtC1y+RnvPrZ/XUqOEDUuxVx27LYlox4wDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:51:11.890184Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.13989","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7abad14695e8d2e3d7fffaccb38db52898e51ff7b98286176652d7e4f56ea797","sha256:8134e4ee68ab989d46062ae6c26f4c686168ce7b4e50d592a864a3d9ce938cf1"],"state_sha256":"d327ac70cb58acbe4619d9d472b6cf4a628ee191194e5dd14b306675344586d0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Axe/F01sblRhu0lBGAUt32PY7yP3k7ImrXCUCcHyS9HlboXuliKEjXv4kRnHgZP9KgF0W3rqhbwcE+D72o6JAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T07:57:49.715743Z","bundle_sha256":"1b04c279ec6f47376f1a202dcbc17634af20a7dd6e31671c9bec4171e16c9711"}}