{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:AFA5YOHTFQEHH55ZNNBGOM7S5U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e"},"schema_version":"1.0","source":{"id":"2205.12191","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"arxiv_version","alias_value":"2205.12191v2","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_12","alias_value":"AFA5YOHTFQEH","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_16","alias_value":"AFA5YOHTFQEHH55Z","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_8","alias_value":"AFA5YOHT","created_at":"2026-07-05T05:56:53Z"}],"graph_snapshots":[{"event_id":"sha256:1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245","target":"graph","created_at":"2026-07-05T05:56:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2205.12191/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Vision-and-language (V&L) models pretrained on large-scale multimodal data have demonstrated strong performance on various tasks such as image captioning and visual question answering (VQA). The quality of such models is commonly assessed by measuring their performance on unseen data that typically comes from the same distribution as the training data. However, when evaluated under out-of-distribution (out-of-dataset) settings for VQA, we observe that these models exhibit poor generalization. We comprehensively evaluate two pretrained V&L models under different settings (i.e. classification an","authors_text":"Aida Nematzadeh, Aishwarya Agrawal, Anita Gergely, Elnaz Davoodi, Emanuele Bugliarello, Ivana Kaji\\'c, Phil Blunsom","cross_cats":["cs.AI","cs.CV","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title":"Reassessing Evaluation Practices in Visual Question Answering: A Case Study on Out-of-Distribution Generalization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.12191","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635","target":"record","created_at":"2026-07-05T05:56:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e"},"schema_version":"1.0","source":{"id":"2205.12191","kind":"arxiv","version":2}},"canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","first_computed_at":"2026-07-05T05:56:53.863692Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:56:53.863692Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/WaoaOZMF8UXoEYLfqVQwNGeNKzBCBanZ4RCEP5oQlL7TDX7GN6hHech1WWUqKx9clue1cY60vmVEjRxA3G5Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T05:56:53.864275Z","signed_message":"canonical_sha256_bytes"},"source_id":"2205.12191","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635","sha256:1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245"],"state_sha256":"e8d9596b034ffb9af3052c5d5dd144f83d66460e683913acd5113d7cc852b9ae"}