{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:Z2APUZQMF2AI3CCG7K2OTZFAMN","short_pith_number":"pith:Z2APUZQM","schema_version":"1.0","canonical_sha256":"ce80fa660c2e808d8846fab4e9e4a0634210233efc280ffa31063e8e59bd27aa","source":{"kind":"arxiv","id":"2410.07112","version":2},"attestation_state":"computed","paper":{"title":"VHELM: A Holistic Evaluation of Vision Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Chi Heem Wong, Cihang Xie, Haoqin Tu, Huaxiu Yao, Josselin Somerville Roberts, Michihiro Yasunaga, Percy Liang, Tony Lee, Wenhao Zheng, Yifan Mai, Yiyang Zhou","submitted_at":"2024-10-09T17:46:34Z","abstract_excerpt":"Current benchmarks for assessing vision-language models (VLMs) often focus on their perception or problem-solving capabilities and neglect other critical aspects such as fairness, multilinguality, or toxicity. Furthermore, they differ in their evaluation procedures and the scope of the evaluation, making it difficult to compare models. To address these issues, we extend the HELM framework to VLMs to present the Holistic Evaluation of Vision Language Models (VHELM). VHELM aggregates various datasets to cover one or more of the 9 aspects: visual perception, knowledge, reasoning, bias, fairness, "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.07112","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-09T17:46:34Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c963088758f9bcceebddc62c8112c0bf049a8ef86f259a20bd450b25c760b7ce","abstract_canon_sha256":"61d59d5d17fbc408eb02e7524c9546e63d36a22381cc6bc1539f831241bae00f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:25:02.068707Z","signature_b64":"i5JFDCHxEcbUeUMqNB+pJfFggQg7K8YMEYrYc5dZ8omQgRqxYZg59FmRO0P9z9YL48BDVJlSUzWgen2jNxIUCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ce80fa660c2e808d8846fab4e9e4a0634210233efc280ffa31063e8e59bd27aa","last_reissued_at":"2026-07-05T09:25:02.068289Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:25:02.068289Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"VHELM: A Holistic Evaluation of Vision Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Chi Heem Wong, Cihang Xie, Haoqin Tu, Huaxiu Yao, Josselin Somerville Roberts, Michihiro Yasunaga, Percy Liang, Tony Lee, Wenhao Zheng, Yifan Mai, Yiyang Zhou","submitted_at":"2024-10-09T17:46:34Z","abstract_excerpt":"Current benchmarks for assessing vision-language models (VLMs) often focus on their perception or problem-solving capabilities and neglect other critical aspects such as fairness, multilinguality, or toxicity. Furthermore, they differ in their evaluation procedures and the scope of the evaluation, making it difficult to compare models. To address these issues, we extend the HELM framework to VLMs to present the Holistic Evaluation of Vision Language Models (VHELM). VHELM aggregates various datasets to cover one or more of the 9 aspects: visual perception, knowledge, reasoning, bias, fairness, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.07112","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.07112/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.07112","created_at":"2026-07-05T09:25:02.068351+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.07112v2","created_at":"2026-07-05T09:25:02.068351+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07112","created_at":"2026-07-05T09:25:02.068351+00:00"},{"alias_kind":"pith_short_12","alias_value":"Z2APUZQMF2AI","created_at":"2026-07-05T09:25:02.068351+00:00"},{"alias_kind":"pith_short_16","alias_value":"Z2APUZQMF2AI3CCG","created_at":"2026-07-05T09:25:02.068351+00:00"},{"alias_kind":"pith_short_8","alias_value":"Z2APUZQM","created_at":"2026-07-05T09:25:02.068351+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.10400","citing_title":"Do Vision-Language Models See or Guess? Measuring and Reducing Textual-Prior Reliance with a Phrasing-Controlled Benchmark","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01513","citing_title":"Compliance-Scored Best-of-N Guardrail Orchestration for Multimodal Document Generation in Payments Dispute Defense","ref_index":21,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN","json":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN.json","graph_json":"https://pith.science/api/pith-number/Z2APUZQMF2AI3CCG7K2OTZFAMN/graph.json","events_json":"https://pith.science/api/pith-number/Z2APUZQMF2AI3CCG7K2OTZFAMN/events.json","paper":"https://pith.science/paper/Z2APUZQM"},"agent_actions":{"view_html":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN","download_json":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN.json","view_paper":"https://pith.science/paper/Z2APUZQM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.07112&json=true","fetch_graph":"https://pith.science/api/pith-number/Z2APUZQMF2AI3CCG7K2OTZFAMN/graph.json","fetch_events":"https://pith.science/api/pith-number/Z2APUZQMF2AI3CCG7K2OTZFAMN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN/action/storage_attestation","attest_author":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN/action/author_attestation","sign_citation":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN/action/citation_signature","submit_replication":"https://pith.science/pith/Z2APUZQMF2AI3CCG7K2OTZFAMN/action/replication_record"}},"created_at":"2026-07-05T09:25:02.068351+00:00","updated_at":"2026-07-05T09:25:02.068351+00:00"}