{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:IYP6PC34XHRMJZDNB3677B6TDN","short_pith_number":"pith:IYP6PC34","canonical_record":{"source":{"id":"2405.02287","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-03T17:59:55Z","cross_cats_sorted":["cs.AI","cs.CV"],"title_canon_sha256":"1d762eb3a110e95d98add2fd44160f8b35e13ede62b9ded54834f50002ac45f7","abstract_canon_sha256":"566e114857dd76978ea16c3ddb2a50111f6a7943d099fa8e784b4a23ef669be7"},"schema_version":"1.0"},"canonical_sha256":"461fe78b7cb9e2c4e46d0efdff87d31b4a997d2bdf037a7c84436df6f6a146ed","source":{"kind":"arxiv","id":"2405.02287","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.02287","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"arxiv_version","alias_value":"2405.02287v1","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.02287","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_12","alias_value":"IYP6PC34XHRM","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_16","alias_value":"IYP6PC34XHRMJZDN","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_8","alias_value":"IYP6PC34","created_at":"2026-07-05T08:15:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:IYP6PC34XHRMJZDNB3677B6TDN","target":"record","payload":{"canonical_record":{"source":{"id":"2405.02287","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-03T17:59:55Z","cross_cats_sorted":["cs.AI","cs.CV"],"title_canon_sha256":"1d762eb3a110e95d98add2fd44160f8b35e13ede62b9ded54834f50002ac45f7","abstract_canon_sha256":"566e114857dd76978ea16c3ddb2a50111f6a7943d099fa8e784b4a23ef669be7"},"schema_version":"1.0"},"canonical_sha256":"461fe78b7cb9e2c4e46d0efdff87d31b4a997d2bdf037a7c84436df6f6a146ed","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:15:09.393298Z","signature_b64":"uApG3H/myZBvypMGr1i3+uztkz+AjuFu4RgwKsCzqQzIXqm76IqQ2IkF86TBzaYtxoza6etZTagQ4TGn7ek8DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"461fe78b7cb9e2c4e46d0efdff87d31b4a997d2bdf037a7c84436df6f6a146ed","last_reissued_at":"2026-07-05T08:15:09.392828Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:15:09.392828Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.02287","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:15:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i1rXbOz+4n8Dqr3mLUKCwVMFR5c4Z7znYX0QKcdZu5W5gjg8ZzQurV0VLQaMOU5zdhDUv3X2g3NDIh1K3eNtAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T17:30:43.059507Z"},"content_sha256":"a869e170ed19bbb8a53fb274dc04ac2e63432847421d0f03ce1c1845d3020ff1","schema_version":"1.0","event_id":"sha256:a869e170ed19bbb8a53fb274dc04ac2e63432847421d0f03ce1c1845d3020ff1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:IYP6PC34XHRMJZDNB3677B6TDN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CV"],"primary_cat":"cs.CL","authors_text":"Aitor Ormazabal, Che Zheng, Cyprien de Masson d'Autume, Dani Yogatama, Deyu Fu, Donovan Ong, Eric Chen, Ethan Yeo, Eugenie Lamprecht, Hai Pham, Kaloyan Aleksiev, Lei Li, Matthew Henderson, Max Bain, Mikel Artetxe, Nishant Relan, Piotr Padlewski, Qi Liu, Samuel Phua, Yi Tay, Yuqi Wang, Zhongkai Zhu","submitted_at":"2024-05-03T17:59:55Z","abstract_excerpt":"We introduce Vibe-Eval: a new open benchmark and framework for evaluating multimodal chat models. Vibe-Eval consists of 269 visual understanding prompts, including 100 of hard difficulty, complete with gold-standard responses authored by experts. Vibe-Eval is open-ended and challenging with dual objectives: (i) vibe checking multimodal chat models for day-to-day tasks and (ii) rigorously testing and probing the capabilities of present frontier models. Notably, our hard set contains >50% questions that all frontier models answer incorrectly. We explore the nuances of designing, evaluating, and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.02287","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.02287/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:15:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+3C5tB8cjHJ16WY/SqOHDrcz36Eo15JBmMzM8Pi37R600mmznR62I0FXLbcsUPXR7M9oahKcdwPHupMc9mTCCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T17:30:43.060012Z"},"content_sha256":"d8de2105702f787697b7e9d0ac1ebea23394aace56896432adef90f479165ad5","schema_version":"1.0","event_id":"sha256:d8de2105702f787697b7e9d0ac1ebea23394aace56896432adef90f479165ad5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IYP6PC34XHRMJZDNB3677B6TDN/bundle.json","state_url":"https://pith.science/pith/IYP6PC34XHRMJZDNB3677B6TDN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IYP6PC34XHRMJZDNB3677B6TDN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T17:30:43Z","links":{"resolver":"https://pith.science/pith/IYP6PC34XHRMJZDNB3677B6TDN","bundle":"https://pith.science/pith/IYP6PC34XHRMJZDNB3677B6TDN/bundle.json","state":"https://pith.science/pith/IYP6PC34XHRMJZDNB3677B6TDN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IYP6PC34XHRMJZDNB3677B6TDN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IYP6PC34XHRMJZDNB3677B6TDN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"566e114857dd76978ea16c3ddb2a50111f6a7943d099fa8e784b4a23ef669be7","cross_cats_sorted":["cs.AI","cs.CV"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-03T17:59:55Z","title_canon_sha256":"1d762eb3a110e95d98add2fd44160f8b35e13ede62b9ded54834f50002ac45f7"},"schema_version":"1.0","source":{"id":"2405.02287","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.02287","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"arxiv_version","alias_value":"2405.02287v1","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.02287","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_12","alias_value":"IYP6PC34XHRM","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_16","alias_value":"IYP6PC34XHRMJZDN","created_at":"2026-07-05T08:15:09Z"},{"alias_kind":"pith_short_8","alias_value":"IYP6PC34","created_at":"2026-07-05T08:15:09Z"}],"graph_snapshots":[{"event_id":"sha256:d8de2105702f787697b7e9d0ac1ebea23394aace56896432adef90f479165ad5","target":"graph","created_at":"2026-07-05T08:15:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.02287/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We introduce Vibe-Eval: a new open benchmark and framework for evaluating multimodal chat models. Vibe-Eval consists of 269 visual understanding prompts, including 100 of hard difficulty, complete with gold-standard responses authored by experts. Vibe-Eval is open-ended and challenging with dual objectives: (i) vibe checking multimodal chat models for day-to-day tasks and (ii) rigorously testing and probing the capabilities of present frontier models. Notably, our hard set contains >50% questions that all frontier models answer incorrectly. We explore the nuances of designing, evaluating, and ","authors_text":"Aitor Ormazabal, Che Zheng, Cyprien de Masson d'Autume, Dani Yogatama, Deyu Fu, Donovan Ong, Eric Chen, Ethan Yeo, Eugenie Lamprecht, Hai Pham, Kaloyan Aleksiev, Lei Li, Matthew Henderson, Max Bain, Mikel Artetxe, Nishant Relan, Piotr Padlewski, Qi Liu, Samuel Phua, Yi Tay, Yuqi Wang, Zhongkai Zhu","cross_cats":["cs.AI","cs.CV"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-03T17:59:55Z","title":"Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.02287","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a869e170ed19bbb8a53fb274dc04ac2e63432847421d0f03ce1c1845d3020ff1","target":"record","created_at":"2026-07-05T08:15:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"566e114857dd76978ea16c3ddb2a50111f6a7943d099fa8e784b4a23ef669be7","cross_cats_sorted":["cs.AI","cs.CV"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-03T17:59:55Z","title_canon_sha256":"1d762eb3a110e95d98add2fd44160f8b35e13ede62b9ded54834f50002ac45f7"},"schema_version":"1.0","source":{"id":"2405.02287","kind":"arxiv","version":1}},"canonical_sha256":"461fe78b7cb9e2c4e46d0efdff87d31b4a997d2bdf037a7c84436df6f6a146ed","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"461fe78b7cb9e2c4e46d0efdff87d31b4a997d2bdf037a7c84436df6f6a146ed","first_computed_at":"2026-07-05T08:15:09.392828Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:15:09.392828Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"uApG3H/myZBvypMGr1i3+uztkz+AjuFu4RgwKsCzqQzIXqm76IqQ2IkF86TBzaYtxoza6etZTagQ4TGn7ek8DA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:15:09.393298Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.02287","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a869e170ed19bbb8a53fb274dc04ac2e63432847421d0f03ce1c1845d3020ff1","sha256:d8de2105702f787697b7e9d0ac1ebea23394aace56896432adef90f479165ad5"],"state_sha256":"510371a4ec5870eb3c24d58038b42e32fb7cd0b816c641a519ec145dec0b96b1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"g3LIfHY12DF9YRd959pHxe3dhfIrltjBR9rr3qRNGRyZGIUnBkmASr5o6+Zil3mAqcdUMvnjkuTu1YJJj/bTAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T17:30:43.064996Z","bundle_sha256":"99c02f5afcb3b890e900a5eee9ddc51d4bb0c4435ef147757c06425a8611d3f7"}}