{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:FBQAX6CYFQ26EQIKUXZMBIEO7F","short_pith_number":"pith:FBQAX6CY","canonical_record":{"source":{"id":"2104.08108","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2021-04-16T13:27:45Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"e1f4df5ad35045c4e8b27de3b2ca94d600a09f732184c0efb8f7074974655de5","abstract_canon_sha256":"10239bef91cab41850856c50d6d5a8ee365c19dc1d16de7efac75dd20df07d4d"},"schema_version":"1.0"},"canonical_sha256":"28600bf8582c35e2410aa5f2c0a08ef94cc0735b3aa99087dd280aca77015d79","source":{"kind":"arxiv","id":"2104.08108","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.08108","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"arxiv_version","alias_value":"2104.08108v1","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.08108","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_12","alias_value":"FBQAX6CYFQ26","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_16","alias_value":"FBQAX6CYFQ26EQIK","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_8","alias_value":"FBQAX6CY","created_at":"2026-07-05T02:32:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:FBQAX6CYFQ26EQIKUXZMBIEO7F","target":"record","payload":{"canonical_record":{"source":{"id":"2104.08108","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2021-04-16T13:27:45Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"e1f4df5ad35045c4e8b27de3b2ca94d600a09f732184c0efb8f7074974655de5","abstract_canon_sha256":"10239bef91cab41850856c50d6d5a8ee365c19dc1d16de7efac75dd20df07d4d"},"schema_version":"1.0"},"canonical_sha256":"28600bf8582c35e2410aa5f2c0a08ef94cc0735b3aa99087dd280aca77015d79","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:32:41.003793Z","signature_b64":"qBFpVpYSRR+Os1G7ITf2OB3xLKktFedeyo39oYPVb+zzLb/BVIUHoBbxAMxv1YXZnPNhsRg7ukXtIiI4PLp9BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"28600bf8582c35e2410aa5f2c0a08ef94cc0735b3aa99087dd280aca77015d79","last_reissued_at":"2026-07-05T02:32:41.003405Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:32:41.003405Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.08108","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:32:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MTMQyg9lyB1SsCYJAnapW0G2AsuZ0MCSPFGFCOfuqlssZKuttYHMeH6LkeCuihhuolLO7LY5Lwj6xgpy7xF5BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:55:50.686953Z"},"content_sha256":"40b1d3be666efd33282a2f5aefcbc662cc395a35dc050f325ae0d6e6220cc2a8","schema_version":"1.0","event_id":"sha256:40b1d3be666efd33282a2f5aefcbc662cc395a35dc050f325ae0d6e6220cc2a8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:FBQAX6CYFQ26EQIKUXZMBIEO7F","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Cross-Modal Retrieval Augmentation for Multi-Modal Classification","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.CV","authors_text":"Austin Reiter, Chris Stauffer, Douwe Kiela, Natalia Neverova, Ser-Nam Lim, Shir Gur","submitted_at":"2021-04-16T13:27:45Z","abstract_excerpt":"Recent advances in using retrieval components over external knowledge sources have shown impressive results for a variety of downstream tasks in natural language processing. Here, we explore the use of unstructured external knowledge sources of images and their corresponding captions for improving visual question answering (VQA). First, we train a novel alignment model for embedding images and captions in the same space, which achieves substantial improvement in performance on image-caption retrieval w.r.t. similar methods. Second, we show that retrieval-augmented multi-modal transformers usin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.08108","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.08108/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:32:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SKiYQIAB43+7m9L/Y14xJySpV1rM//aRUOzstw3u9Xo4HMG6UsKr/BsPMDncJp/Pw8zbigBOOAcAaIk/T/AEAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:55:50.687341Z"},"content_sha256":"ce909b5895528eede71810bf0c3af054e64aaab58c73febb4be80ea8f143c0a4","schema_version":"1.0","event_id":"sha256:ce909b5895528eede71810bf0c3af054e64aaab58c73febb4be80ea8f143c0a4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/bundle.json","state_url":"https://pith.science/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T06:55:50Z","links":{"resolver":"https://pith.science/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F","bundle":"https://pith.science/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/bundle.json","state":"https://pith.science/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FBQAX6CYFQ26EQIKUXZMBIEO7F/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:FBQAX6CYFQ26EQIKUXZMBIEO7F","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"10239bef91cab41850856c50d6d5a8ee365c19dc1d16de7efac75dd20df07d4d","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2021-04-16T13:27:45Z","title_canon_sha256":"e1f4df5ad35045c4e8b27de3b2ca94d600a09f732184c0efb8f7074974655de5"},"schema_version":"1.0","source":{"id":"2104.08108","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.08108","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"arxiv_version","alias_value":"2104.08108v1","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.08108","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_12","alias_value":"FBQAX6CYFQ26","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_16","alias_value":"FBQAX6CYFQ26EQIK","created_at":"2026-07-05T02:32:41Z"},{"alias_kind":"pith_short_8","alias_value":"FBQAX6CY","created_at":"2026-07-05T02:32:41Z"}],"graph_snapshots":[{"event_id":"sha256:ce909b5895528eede71810bf0c3af054e64aaab58c73febb4be80ea8f143c0a4","target":"graph","created_at":"2026-07-05T02:32:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.08108/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advances in using retrieval components over external knowledge sources have shown impressive results for a variety of downstream tasks in natural language processing. Here, we explore the use of unstructured external knowledge sources of images and their corresponding captions for improving visual question answering (VQA). First, we train a novel alignment model for embedding images and captions in the same space, which achieves substantial improvement in performance on image-caption retrieval w.r.t. similar methods. Second, we show that retrieval-augmented multi-modal transformers usin","authors_text":"Austin Reiter, Chris Stauffer, Douwe Kiela, Natalia Neverova, Ser-Nam Lim, Shir Gur","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2021-04-16T13:27:45Z","title":"Cross-Modal Retrieval Augmentation for Multi-Modal Classification"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.08108","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:40b1d3be666efd33282a2f5aefcbc662cc395a35dc050f325ae0d6e6220cc2a8","target":"record","created_at":"2026-07-05T02:32:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"10239bef91cab41850856c50d6d5a8ee365c19dc1d16de7efac75dd20df07d4d","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2021-04-16T13:27:45Z","title_canon_sha256":"e1f4df5ad35045c4e8b27de3b2ca94d600a09f732184c0efb8f7074974655de5"},"schema_version":"1.0","source":{"id":"2104.08108","kind":"arxiv","version":1}},"canonical_sha256":"28600bf8582c35e2410aa5f2c0a08ef94cc0735b3aa99087dd280aca77015d79","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"28600bf8582c35e2410aa5f2c0a08ef94cc0735b3aa99087dd280aca77015d79","first_computed_at":"2026-07-05T02:32:41.003405Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:32:41.003405Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qBFpVpYSRR+Os1G7ITf2OB3xLKktFedeyo39oYPVb+zzLb/BVIUHoBbxAMxv1YXZnPNhsRg7ukXtIiI4PLp9BA==","signature_status":"signed_v1","signed_at":"2026-07-05T02:32:41.003793Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.08108","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:40b1d3be666efd33282a2f5aefcbc662cc395a35dc050f325ae0d6e6220cc2a8","sha256:ce909b5895528eede71810bf0c3af054e64aaab58c73febb4be80ea8f143c0a4"],"state_sha256":"a7c9df2faee3f831f632df2b553038e2d46f6cfe69319df3f26197257c4ef022"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rLZRIlTnoBJAHX4MbxU8qqBxe714fOqq6dsqwv7uB4RNo6rSL789jyWav7vk9UwpEE5fhbGJMp4UYBFFFZwmDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T06:55:50.689287Z","bundle_sha256":"d0f3ad42b9090a3f33a2e969c5e794dcf7351409bd6aa3ff705be555dedfd4b2"}}