{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:6PTZFORRHJXQ76M4LPKZHONUVJ","short_pith_number":"pith:6PTZFORR","canonical_record":{"source":{"id":"2501.09672","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-01-16T17:08:12Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"513d4084d6f1ced6742826f661c245718d2c526a57c855d6490a81b975088c54","abstract_canon_sha256":"d6734d456fa8eea850280babef129b81614bd9aff2635b95dfef8bd019c71528"},"schema_version":"1.0"},"canonical_sha256":"f3e792ba313a6f0ff99c5bd593b9b4aa68a1ee06b0daf269822c0815ddc3d1e2","source":{"kind":"arxiv","id":"2501.09672","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.09672","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"arxiv_version","alias_value":"2501.09672v3","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.09672","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_12","alias_value":"6PTZFORRHJXQ","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_16","alias_value":"6PTZFORRHJXQ76M4","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_8","alias_value":"6PTZFORR","created_at":"2026-07-05T11:48:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:6PTZFORRHJXQ76M4LPKZHONUVJ","target":"record","payload":{"canonical_record":{"source":{"id":"2501.09672","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-01-16T17:08:12Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"513d4084d6f1ced6742826f661c245718d2c526a57c855d6490a81b975088c54","abstract_canon_sha256":"d6734d456fa8eea850280babef129b81614bd9aff2635b95dfef8bd019c71528"},"schema_version":"1.0"},"canonical_sha256":"f3e792ba313a6f0ff99c5bd593b9b4aa68a1ee06b0daf269822c0815ddc3d1e2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:48:22.433470Z","signature_b64":"H1EysILIqvBO7egp3IyxnYx6MVFnfnC89drzcI9OouhW9/cIvIaldMkkoUIQzsdpBOWEyj6aVkZYgeE2Z0x3CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f3e792ba313a6f0ff99c5bd593b9b4aa68a1ee06b0daf269822c0815ddc3d1e2","last_reissued_at":"2026-07-05T11:48:22.432969Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:48:22.432969Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2501.09672","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:48:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Y0TUU4mJhd0v9Q1mxWUyo4hIkOxCUK+UiSIIZDAVE04Q1OrE07kvQG+MGOfieN5E5dY1Sp7WS6wkjYmKiQ3hAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T13:41:55.357649Z"},"content_sha256":"972afcb428a57f32a826bba151048de7250fe286b46b7109032b5bb028324e75","schema_version":"1.0","event_id":"sha256:972afcb428a57f32a826bba151048de7250fe286b46b7109032b5bb028324e75"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:6PTZFORRHJXQ76M4LPKZHONUVJ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CHIRP: A Fine-Grained Benchmark for Open-Ended Response Evaluation in Vision-Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Alexis Roger, Daniel Z. Kaplan, Edwin Fennell, George Adamopoulos, Irina Rish, Jonathan Siu Chi Lim, Kshitij Gupta, Prateek Humane, Qi Sun, Quentin Anthony","submitted_at":"2025-01-16T17:08:12Z","abstract_excerpt":"The proliferation of Vision-Language Models (VLMs) in the past several years calls for rigorous and comprehensive evaluation methods and benchmarks. This work analyzes existing VLM evaluation techniques, including automated metrics, AI-based assessments, and human evaluations across diverse tasks. We first introduce Robin - a novel suite of VLMs that we built by combining Large Language Models (LLMs) and Vision Encoders (VEs) at multiple scales, and use Robin to identify shortcomings of current evaluation approaches across scales. Next, to overcome the identified limitations, we introduce CHIR"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.09672","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.09672/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:48:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZJCBX2D6rix+zm3fXGrxf+4H9g6XHcZIDMomFu73Swfn7rL6byzwmYO4rK42JWPdNmICsdsnypeZlMdZy2t0Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T13:41:55.358158Z"},"content_sha256":"43c0288ebef270728a4e55b9dfaf171964baffad81e047aa0f796975cff5dc93","schema_version":"1.0","event_id":"sha256:43c0288ebef270728a4e55b9dfaf171964baffad81e047aa0f796975cff5dc93"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/bundle.json","state_url":"https://pith.science/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T13:41:55Z","links":{"resolver":"https://pith.science/pith/6PTZFORRHJXQ76M4LPKZHONUVJ","bundle":"https://pith.science/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/bundle.json","state":"https://pith.science/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6PTZFORRHJXQ76M4LPKZHONUVJ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:6PTZFORRHJXQ76M4LPKZHONUVJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d6734d456fa8eea850280babef129b81614bd9aff2635b95dfef8bd019c71528","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-01-16T17:08:12Z","title_canon_sha256":"513d4084d6f1ced6742826f661c245718d2c526a57c855d6490a81b975088c54"},"schema_version":"1.0","source":{"id":"2501.09672","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.09672","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"arxiv_version","alias_value":"2501.09672v3","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.09672","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_12","alias_value":"6PTZFORRHJXQ","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_16","alias_value":"6PTZFORRHJXQ76M4","created_at":"2026-07-05T11:48:22Z"},{"alias_kind":"pith_short_8","alias_value":"6PTZFORR","created_at":"2026-07-05T11:48:22Z"}],"graph_snapshots":[{"event_id":"sha256:43c0288ebef270728a4e55b9dfaf171964baffad81e047aa0f796975cff5dc93","target":"graph","created_at":"2026-07-05T11:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.09672/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The proliferation of Vision-Language Models (VLMs) in the past several years calls for rigorous and comprehensive evaluation methods and benchmarks. This work analyzes existing VLM evaluation techniques, including automated metrics, AI-based assessments, and human evaluations across diverse tasks. We first introduce Robin - a novel suite of VLMs that we built by combining Large Language Models (LLMs) and Vision Encoders (VEs) at multiple scales, and use Robin to identify shortcomings of current evaluation approaches across scales. Next, to overcome the identified limitations, we introduce CHIR","authors_text":"Alexis Roger, Daniel Z. Kaplan, Edwin Fennell, George Adamopoulos, Irina Rish, Jonathan Siu Chi Lim, Kshitij Gupta, Prateek Humane, Qi Sun, Quentin Anthony","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-01-16T17:08:12Z","title":"CHIRP: A Fine-Grained Benchmark for Open-Ended Response Evaluation in Vision-Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.09672","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:972afcb428a57f32a826bba151048de7250fe286b46b7109032b5bb028324e75","target":"record","created_at":"2026-07-05T11:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d6734d456fa8eea850280babef129b81614bd9aff2635b95dfef8bd019c71528","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-01-16T17:08:12Z","title_canon_sha256":"513d4084d6f1ced6742826f661c245718d2c526a57c855d6490a81b975088c54"},"schema_version":"1.0","source":{"id":"2501.09672","kind":"arxiv","version":3}},"canonical_sha256":"f3e792ba313a6f0ff99c5bd593b9b4aa68a1ee06b0daf269822c0815ddc3d1e2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f3e792ba313a6f0ff99c5bd593b9b4aa68a1ee06b0daf269822c0815ddc3d1e2","first_computed_at":"2026-07-05T11:48:22.432969Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:48:22.432969Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"H1EysILIqvBO7egp3IyxnYx6MVFnfnC89drzcI9OouhW9/cIvIaldMkkoUIQzsdpBOWEyj6aVkZYgeE2Z0x3CA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:48:22.433470Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.09672","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:972afcb428a57f32a826bba151048de7250fe286b46b7109032b5bb028324e75","sha256:43c0288ebef270728a4e55b9dfaf171964baffad81e047aa0f796975cff5dc93"],"state_sha256":"da75fd267c32ae567d683c3c613eab06c71dd7345117575bdd12fa6e50c79b32"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4o6lFuj4hUYFQwbwwQdMXjfUyAwOJuseedXXpHfed7TVP/1xqA8JD+A+6vY37xOkxUG2qNipon9K6vRKpohbDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T13:41:55.362201Z","bundle_sha256":"87e0e6a845e705cd5b6c2a59858e64f3f27e4f4826b20e46ef65a23ca1e39e77"}}