{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:G32AW4F67OJJOFY5J72Z47GJPH","short_pith_number":"pith:G32AW4F6","canonical_record":{"source":{"id":"2410.14669","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-18T17:58:21Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"eb18c98fd3c6805060b3539a8e8876ee4655812188fed9206059e17263d3dbeb","abstract_canon_sha256":"a4ceb7219139cb11894a17a62603d70c2d2f9ab1d0b8c69dbacda000bf160228"},"schema_version":"1.0"},"canonical_sha256":"36f40b70befb9297171d4ff59e7cc979ff5501be1f1973fb8f95477a84da4d57","source":{"kind":"arxiv","id":"2410.14669","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.14669","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"arxiv_version","alias_value":"2410.14669v4","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.14669","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_12","alias_value":"G32AW4F67OJJ","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_16","alias_value":"G32AW4F67OJJOFY5","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_8","alias_value":"G32AW4F6","created_at":"2026-07-05T11:19:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:G32AW4F67OJJOFY5J72Z47GJPH","target":"record","payload":{"canonical_record":{"source":{"id":"2410.14669","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-18T17:58:21Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"eb18c98fd3c6805060b3539a8e8876ee4655812188fed9206059e17263d3dbeb","abstract_canon_sha256":"a4ceb7219139cb11894a17a62603d70c2d2f9ab1d0b8c69dbacda000bf160228"},"schema_version":"1.0"},"canonical_sha256":"36f40b70befb9297171d4ff59e7cc979ff5501be1f1973fb8f95477a84da4d57","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:19:01.056135Z","signature_b64":"LJCkquMsbABsxfPM+Y8hVH+yo79eadH94ELq1aY2pyN0GRwFnjDCAHk0glAr1SznRmIpTr3sA/JVM3o/PLHJCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"36f40b70befb9297171d4ff59e7cc979ff5501be1f1973fb8f95477a84da4d57","last_reissued_at":"2026-07-05T11:19:01.055669Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:19:01.055669Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.14669","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CJyA/G62koARCX1bzlmXUE9EAaT7yICj4ahrCTd7smZS6eKVCC4dk0ZQcI6o/xkSg2zGmY2UjyDHlseLkVJFDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:02:58.364873Z"},"content_sha256":"e6d465e689542868cdb676de60de84eefa75062dea220a74a6eecf77e13b3b69","schema_version":"1.0","event_id":"sha256:e6d465e689542868cdb676de60de84eefa75062dea220a74a6eecf77e13b3b69"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:G32AW4F67OJJOFY5J72Z47GJPH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"NaturalBench: Evaluating Vision-Language Models on Natural Adversarial Samples","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.CV","authors_text":"Baiqi Li, Daniel Jiang, Deva Ramanan, Graham Neubig, Jean de Dieu Nyandwi, Ranjay Krishna, Simran Khanuja, Wenxuan Peng, Zhiqiu Lin, Zixian Ma","submitted_at":"2024-10-18T17:58:21Z","abstract_excerpt":"Vision-language models (VLMs) have made significant progress in recent visual-question-answering (VQA) benchmarks that evaluate complex visio-linguistic reasoning. However, are these models truly effective? In this work, we show that VLMs still struggle with natural images and questions that humans can easily answer, which we term natural adversarial samples. We also find it surprisingly easy to generate these VQA samples from natural image-text corpora using off-the-shelf models like CLIP and ChatGPT. We propose a semi-automated approach to collect a new benchmark, NaturalBench, for reliably "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.14669","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.14669/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VBRUHRM7gw3Do2XGBoVx0jpTvfr8rqcmPyalc7bGiLibFfWkeyM4859F1lUPw5RIa02bgXzFS2xwLRAzE+gHDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:02:58.365594Z"},"content_sha256":"586429dac30a2a4b371ce25143b4392d6dac753b1f748a3b4d57c30e48f022e7","schema_version":"1.0","event_id":"sha256:586429dac30a2a4b371ce25143b4392d6dac753b1f748a3b4d57c30e48f022e7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/G32AW4F67OJJOFY5J72Z47GJPH/bundle.json","state_url":"https://pith.science/pith/G32AW4F67OJJOFY5J72Z47GJPH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/G32AW4F67OJJOFY5J72Z47GJPH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T01:02:58Z","links":{"resolver":"https://pith.science/pith/G32AW4F67OJJOFY5J72Z47GJPH","bundle":"https://pith.science/pith/G32AW4F67OJJOFY5J72Z47GJPH/bundle.json","state":"https://pith.science/pith/G32AW4F67OJJOFY5J72Z47GJPH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/G32AW4F67OJJOFY5J72Z47GJPH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:G32AW4F67OJJOFY5J72Z47GJPH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a4ceb7219139cb11894a17a62603d70c2d2f9ab1d0b8c69dbacda000bf160228","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-18T17:58:21Z","title_canon_sha256":"eb18c98fd3c6805060b3539a8e8876ee4655812188fed9206059e17263d3dbeb"},"schema_version":"1.0","source":{"id":"2410.14669","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.14669","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"arxiv_version","alias_value":"2410.14669v4","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.14669","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_12","alias_value":"G32AW4F67OJJ","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_16","alias_value":"G32AW4F67OJJOFY5","created_at":"2026-07-05T11:19:01Z"},{"alias_kind":"pith_short_8","alias_value":"G32AW4F6","created_at":"2026-07-05T11:19:01Z"}],"graph_snapshots":[{"event_id":"sha256:586429dac30a2a4b371ce25143b4392d6dac753b1f748a3b4d57c30e48f022e7","target":"graph","created_at":"2026-07-05T11:19:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.14669/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Vision-language models (VLMs) have made significant progress in recent visual-question-answering (VQA) benchmarks that evaluate complex visio-linguistic reasoning. However, are these models truly effective? In this work, we show that VLMs still struggle with natural images and questions that humans can easily answer, which we term natural adversarial samples. We also find it surprisingly easy to generate these VQA samples from natural image-text corpora using off-the-shelf models like CLIP and ChatGPT. We propose a semi-automated approach to collect a new benchmark, NaturalBench, for reliably ","authors_text":"Baiqi Li, Daniel Jiang, Deva Ramanan, Graham Neubig, Jean de Dieu Nyandwi, Ranjay Krishna, Simran Khanuja, Wenxuan Peng, Zhiqiu Lin, Zixian Ma","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-18T17:58:21Z","title":"NaturalBench: Evaluating Vision-Language Models on Natural Adversarial Samples"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.14669","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e6d465e689542868cdb676de60de84eefa75062dea220a74a6eecf77e13b3b69","target":"record","created_at":"2026-07-05T11:19:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a4ceb7219139cb11894a17a62603d70c2d2f9ab1d0b8c69dbacda000bf160228","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-10-18T17:58:21Z","title_canon_sha256":"eb18c98fd3c6805060b3539a8e8876ee4655812188fed9206059e17263d3dbeb"},"schema_version":"1.0","source":{"id":"2410.14669","kind":"arxiv","version":4}},"canonical_sha256":"36f40b70befb9297171d4ff59e7cc979ff5501be1f1973fb8f95477a84da4d57","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"36f40b70befb9297171d4ff59e7cc979ff5501be1f1973fb8f95477a84da4d57","first_computed_at":"2026-07-05T11:19:01.055669Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:19:01.055669Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LJCkquMsbABsxfPM+Y8hVH+yo79eadH94ELq1aY2pyN0GRwFnjDCAHk0glAr1SznRmIpTr3sA/JVM3o/PLHJCA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:19:01.056135Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.14669","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e6d465e689542868cdb676de60de84eefa75062dea220a74a6eecf77e13b3b69","sha256:586429dac30a2a4b371ce25143b4392d6dac753b1f748a3b4d57c30e48f022e7"],"state_sha256":"56aa0a62257691d1ff15e6a3e5ebc027bf2fe719ca4a405734f732dd2baa8ca6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"U3YMmikQBsenEudAq+ooCsqVJbNN1krdJgwnZYYkyp0E16lXQG7hK1lQeC61VYbFhpf53RgoWrLImsXVoSQfCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T01:02:58.372494Z","bundle_sha256":"71f76fcc000b938eb99ba7a9bfbe18807b8c2d9b0341decb84af741728af757f"}}