{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:AFA5YOHTFQEHH55ZNNBGOM7S5U","short_pith_number":"pith:AFA5YOHT","canonical_record":{"source":{"id":"2205.12191","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e","abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d"},"schema_version":"1.0"},"canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","source":{"kind":"arxiv","id":"2205.12191","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"arxiv_version","alias_value":"2205.12191v2","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_12","alias_value":"AFA5YOHTFQEH","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_16","alias_value":"AFA5YOHTFQEHH55Z","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_8","alias_value":"AFA5YOHT","created_at":"2026-07-05T05:56:53Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:AFA5YOHTFQEHH55ZNNBGOM7S5U","target":"record","payload":{"canonical_record":{"source":{"id":"2205.12191","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e","abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d"},"schema_version":"1.0"},"canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:56:53.864275Z","signature_b64":"/WaoaOZMF8UXoEYLfqVQwNGeNKzBCBanZ4RCEP5oQlL7TDX7GN6hHech1WWUqKx9clue1cY60vmVEjRxA3G5Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","last_reissued_at":"2026-07-05T05:56:53.863692Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:56:53.863692Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2205.12191","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:56:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CqubkfiyhOuV0e0KmKuTKOZBpXpgiVq17TGR6ajPWwOdhXoMygY+XQyCN+B7YeGATnGEkxvBV0lyLsi2JR6hDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T04:17:42.045360Z"},"content_sha256":"85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635","schema_version":"1.0","event_id":"sha256:85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:AFA5YOHTFQEHH55ZNNBGOM7S5U","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reassessing Evaluation Practices in Visual Question Answering: A Case Study on Out-of-Distribution Generalization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CV","cs.LG"],"primary_cat":"cs.CL","authors_text":"Aida Nematzadeh, Aishwarya Agrawal, Anita Gergely, Elnaz Davoodi, Emanuele Bugliarello, Ivana Kaji\\'c, Phil Blunsom","submitted_at":"2022-05-24T16:44:45Z","abstract_excerpt":"Vision-and-language (V&L) models pretrained on large-scale multimodal data have demonstrated strong performance on various tasks such as image captioning and visual question answering (VQA). The quality of such models is commonly assessed by measuring their performance on unseen data that typically comes from the same distribution as the training data. However, when evaluated under out-of-distribution (out-of-dataset) settings for VQA, we observe that these models exhibit poor generalization. We comprehensively evaluate two pretrained V&L models under different settings (i.e. classification an"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.12191","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2205.12191/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:56:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D5/RuCREuXQfP2csn26Mkr57XxFxUxu/mEqwZCD/Gs0h6Pd7vNC68AFK9KjyPpjf28Z86HozmvdY/gaywI/0DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T04:17:42.045900Z"},"content_sha256":"1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245","schema_version":"1.0","event_id":"sha256:1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/bundle.json","state_url":"https://pith.science/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T04:17:42Z","links":{"resolver":"https://pith.science/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U","bundle":"https://pith.science/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/bundle.json","state":"https://pith.science/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AFA5YOHTFQEHH55ZNNBGOM7S5U/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:AFA5YOHTFQEHH55ZNNBGOM7S5U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e"},"schema_version":"1.0","source":{"id":"2205.12191","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"arxiv_version","alias_value":"2205.12191v2","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.12191","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_12","alias_value":"AFA5YOHTFQEH","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_16","alias_value":"AFA5YOHTFQEHH55Z","created_at":"2026-07-05T05:56:53Z"},{"alias_kind":"pith_short_8","alias_value":"AFA5YOHT","created_at":"2026-07-05T05:56:53Z"}],"graph_snapshots":[{"event_id":"sha256:1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245","target":"graph","created_at":"2026-07-05T05:56:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2205.12191/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Vision-and-language (V&L) models pretrained on large-scale multimodal data have demonstrated strong performance on various tasks such as image captioning and visual question answering (VQA). The quality of such models is commonly assessed by measuring their performance on unseen data that typically comes from the same distribution as the training data. However, when evaluated under out-of-distribution (out-of-dataset) settings for VQA, we observe that these models exhibit poor generalization. We comprehensively evaluate two pretrained V&L models under different settings (i.e. classification an","authors_text":"Aida Nematzadeh, Aishwarya Agrawal, Anita Gergely, Elnaz Davoodi, Emanuele Bugliarello, Ivana Kaji\\'c, Phil Blunsom","cross_cats":["cs.AI","cs.CV","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title":"Reassessing Evaluation Practices in Visual Question Answering: A Case Study on Out-of-Distribution Generalization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.12191","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635","target":"record","created_at":"2026-07-05T05:56:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4edcf815a162d6df0bf96bef5d2af19c0368bc7472c7a989bed7b041e48e250d","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-24T16:44:45Z","title_canon_sha256":"cb48dfe0110d0f7efb5a2e35283d1f7c8e8d9987164a43353735e9e52a979d9e"},"schema_version":"1.0","source":{"id":"2205.12191","kind":"arxiv","version":2}},"canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0141dc38f32c0873f7b96b426733f2ed24445a5e853010356a297a07c0684fca","first_computed_at":"2026-07-05T05:56:53.863692Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:56:53.863692Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/WaoaOZMF8UXoEYLfqVQwNGeNKzBCBanZ4RCEP5oQlL7TDX7GN6hHech1WWUqKx9clue1cY60vmVEjRxA3G5Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T05:56:53.864275Z","signed_message":"canonical_sha256_bytes"},"source_id":"2205.12191","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:85a0d9d9a2e6e462c5b2e0043db48b0e124ea7937f0006b6d6d241d1367cd635","sha256:1ee2583ecdf0c8a2f5e079ef3e985f6559cf7228da08bab2be96d72ed97fd245"],"state_sha256":"e8d9596b034ffb9af3052c5d5dd144f83d66460e683913acd5113d7cc852b9ae"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fbZa2O5N81bgV1mUj71h6zhU6cPtX43q9fzG1q6Bd9KoU35kegt7xLWCPfvvL2GALgI5N5DUjjmBnLcXDwgcDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T04:17:42.049924Z","bundle_sha256":"4be3b6464ba260126cc167090dd0918db1ec44ea61f144f98985d01653cedca7"}}