{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:3HBEY4L6EKHJVSN52BHIUMHP3U","short_pith_number":"pith:3HBEY4L6","canonical_record":{"source":{"id":"2305.18786","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-05-30T06:40:08Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"cf38b3582d79590a3f6546ef0fe8ce4b90e0411c208e6a4339385ee9edee1ded","abstract_canon_sha256":"16f2b555294943ade56476e0565547a89656f03ddae6508532525fe0b933a41e"},"schema_version":"1.0"},"canonical_sha256":"d9c24c717e228e9ac9bdd04e8a30efdd29bb0942b1c8e6db41629ea3e0328298","source":{"kind":"arxiv","id":"2305.18786","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.18786","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"arxiv_version","alias_value":"2305.18786v2","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.18786","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_12","alias_value":"3HBEY4L6EKHJ","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_16","alias_value":"3HBEY4L6EKHJVSN5","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_8","alias_value":"3HBEY4L6","created_at":"2026-07-05T06:16:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:3HBEY4L6EKHJVSN52BHIUMHP3U","target":"record","payload":{"canonical_record":{"source":{"id":"2305.18786","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-05-30T06:40:08Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"cf38b3582d79590a3f6546ef0fe8ce4b90e0411c208e6a4339385ee9edee1ded","abstract_canon_sha256":"16f2b555294943ade56476e0565547a89656f03ddae6508532525fe0b933a41e"},"schema_version":"1.0"},"canonical_sha256":"d9c24c717e228e9ac9bdd04e8a30efdd29bb0942b1c8e6db41629ea3e0328298","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:16:02.875247Z","signature_b64":"7tw76VceOWcuZFw4+OKYYxg4fz0K45IfQRHsM8fy+ZB2aaR7iowQkOc8hvVSRo3nu9KrHPLSWk7kbuTIiBdrAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d9c24c717e228e9ac9bdd04e8a30efdd29bb0942b1c8e6db41629ea3e0328298","last_reissued_at":"2026-07-05T06:16:02.874753Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:16:02.874753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.18786","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:16:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QCbP1wKAh/tqgCHsMn74Y5Ox34ckJw3jGEgnwBOdsmRUDsN8awEXvmzak/t86A44tbOqwoI9LkljHDtFkTjIBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T10:30:12.803332Z"},"content_sha256":"64ff0f08080dbd29882e51bb78d4418d0b0426bcb3cc60659c53039702c1d533","schema_version":"1.0","event_id":"sha256:64ff0f08080dbd29882e51bb78d4418d0b0426bcb3cc60659c53039702c1d533"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:3HBEY4L6EKHJVSN52BHIUMHP3U","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Scalable Performance Analysis for Vision-Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.CV","authors_text":"Oana Ignat, Rada Mihalcea, Santiago Castro","submitted_at":"2023-05-30T06:40:08Z","abstract_excerpt":"Joint vision-language models have shown great performance over a diverse set of tasks. However, little is known about their limitations, as the high dimensional space learned by these models makes it difficult to identify semantic errors. Recent work has addressed this problem by designing highly controlled probing task benchmarks. Our paper introduces a more scalable solution that relies on already annotated benchmarks. Our method consists of extracting a large set of diverse features from a vision-language benchmark and measuring their correlation with the output of the target model. We conf"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.18786","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.18786/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:16:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Nmhhc2TF5PiJFdy1V6MKOMRXl2RL9s9GG1sRow9CW4Ek9xdX8k9NY6FsxKUpsP4I4UzQ8N/ge5PuuLJqFuntBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T10:30:12.803839Z"},"content_sha256":"68150a4f33b8c044f45347cf2c87aac5055aa46f073c6f77dc901493de197e48","schema_version":"1.0","event_id":"sha256:68150a4f33b8c044f45347cf2c87aac5055aa46f073c6f77dc901493de197e48"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/bundle.json","state_url":"https://pith.science/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T10:30:12Z","links":{"resolver":"https://pith.science/pith/3HBEY4L6EKHJVSN52BHIUMHP3U","bundle":"https://pith.science/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/bundle.json","state":"https://pith.science/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3HBEY4L6EKHJVSN52BHIUMHP3U/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:3HBEY4L6EKHJVSN52BHIUMHP3U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"16f2b555294943ade56476e0565547a89656f03ddae6508532525fe0b933a41e","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-05-30T06:40:08Z","title_canon_sha256":"cf38b3582d79590a3f6546ef0fe8ce4b90e0411c208e6a4339385ee9edee1ded"},"schema_version":"1.0","source":{"id":"2305.18786","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.18786","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"arxiv_version","alias_value":"2305.18786v2","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.18786","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_12","alias_value":"3HBEY4L6EKHJ","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_16","alias_value":"3HBEY4L6EKHJVSN5","created_at":"2026-07-05T06:16:02Z"},{"alias_kind":"pith_short_8","alias_value":"3HBEY4L6","created_at":"2026-07-05T06:16:02Z"}],"graph_snapshots":[{"event_id":"sha256:68150a4f33b8c044f45347cf2c87aac5055aa46f073c6f77dc901493de197e48","target":"graph","created_at":"2026-07-05T06:16:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.18786/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Joint vision-language models have shown great performance over a diverse set of tasks. However, little is known about their limitations, as the high dimensional space learned by these models makes it difficult to identify semantic errors. Recent work has addressed this problem by designing highly controlled probing task benchmarks. Our paper introduces a more scalable solution that relies on already annotated benchmarks. Our method consists of extracting a large set of diverse features from a vision-language benchmark and measuring their correlation with the output of the target model. We conf","authors_text":"Oana Ignat, Rada Mihalcea, Santiago Castro","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-05-30T06:40:08Z","title":"Scalable Performance Analysis for Vision-Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.18786","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:64ff0f08080dbd29882e51bb78d4418d0b0426bcb3cc60659c53039702c1d533","target":"record","created_at":"2026-07-05T06:16:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"16f2b555294943ade56476e0565547a89656f03ddae6508532525fe0b933a41e","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-05-30T06:40:08Z","title_canon_sha256":"cf38b3582d79590a3f6546ef0fe8ce4b90e0411c208e6a4339385ee9edee1ded"},"schema_version":"1.0","source":{"id":"2305.18786","kind":"arxiv","version":2}},"canonical_sha256":"d9c24c717e228e9ac9bdd04e8a30efdd29bb0942b1c8e6db41629ea3e0328298","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d9c24c717e228e9ac9bdd04e8a30efdd29bb0942b1c8e6db41629ea3e0328298","first_computed_at":"2026-07-05T06:16:02.874753Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:16:02.874753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"7tw76VceOWcuZFw4+OKYYxg4fz0K45IfQRHsM8fy+ZB2aaR7iowQkOc8hvVSRo3nu9KrHPLSWk7kbuTIiBdrAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:16:02.875247Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.18786","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:64ff0f08080dbd29882e51bb78d4418d0b0426bcb3cc60659c53039702c1d533","sha256:68150a4f33b8c044f45347cf2c87aac5055aa46f073c6f77dc901493de197e48"],"state_sha256":"14fe90dede114c79c82d8ad6c9c494d17a8fc740bfb85c190d52e00f649c2dcc"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"as6lwftKPsjyzo2zdEcIUHSD2GDTVQUC6U1zyYOs1s/4ZVSLdeJIp+RsdcKXOtYNnHOa5VaIJag7gYrGMWSjDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T10:30:12.807714Z","bundle_sha256":"d3c8ddc8a7f14b43e8c6d39bc8aab3b92d3702fa7104bc537e5ca131f01fb4ca"}}