{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:LSR2E3ISP5LAY2TT3QBKSLWU6B","short_pith_number":"pith:LSR2E3IS","canonical_record":{"source":{"id":"2310.00647","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-10-01T12:02:59Z","cross_cats_sorted":["cs.MM"],"title_canon_sha256":"48b2a39392920bbc837720e26a003ed2b3f3fe2af5f2ec94d191ee3530644d98","abstract_canon_sha256":"cad2663cbcde5d3e26a840c0dcf54923f102fe07d54a0b8d917f7f801680ac61"},"schema_version":"1.0"},"canonical_sha256":"5ca3a26d127f560c6a73dc02a92ed4f07a603ad38e6558bebeca478b74da4e22","source":{"kind":"arxiv","id":"2310.00647","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.00647","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"arxiv_version","alias_value":"2310.00647v2","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.00647","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_12","alias_value":"LSR2E3ISP5LA","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_16","alias_value":"LSR2E3ISP5LAY2TT","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_8","alias_value":"LSR2E3IS","created_at":"2026-07-05T07:36:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:LSR2E3ISP5LAY2TT3QBKSLWU6B","target":"record","payload":{"canonical_record":{"source":{"id":"2310.00647","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-10-01T12:02:59Z","cross_cats_sorted":["cs.MM"],"title_canon_sha256":"48b2a39392920bbc837720e26a003ed2b3f3fe2af5f2ec94d191ee3530644d98","abstract_canon_sha256":"cad2663cbcde5d3e26a840c0dcf54923f102fe07d54a0b8d917f7f801680ac61"},"schema_version":"1.0"},"canonical_sha256":"5ca3a26d127f560c6a73dc02a92ed4f07a603ad38e6558bebeca478b74da4e22","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:36:03.391476Z","signature_b64":"TURCEIThVu8ZjHPnmNl24bbGRfuDRuynOPdHBTG6+Z5+b1Fj8mUASA6g/I3H4Ap75FKHbmoUXH95d64d6qi+Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5ca3a26d127f560c6a73dc02a92ed4f07a603ad38e6558bebeca478b74da4e22","last_reissued_at":"2026-07-05T07:36:03.390974Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:36:03.390974Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.00647","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:36:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UjjpOHFiNV0TFigvb5pKiaK+RwDr4XX7tecIBm6ZoVw7o8WqK5M6ZNU1IfnOC0TtqDLZt6wrdRlCX9HFDc5EBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T19:27:18.365863Z"},"content_sha256":"fd443f24fce514c207b44e79a5213c38cc22b22ddd0362fab26694a1209ab122","schema_version":"1.0","event_id":"sha256:fd443f24fce514c207b44e79a5213c38cc22b22ddd0362fab26694a1209ab122"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:LSR2E3ISP5LAY2TT3QBKSLWU6B","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Task Performance: Evaluating and Reducing the Flaws of Large Multimodal Models with In-Context Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.MM"],"primary_cat":"cs.CV","authors_text":"Alexandre Rame, Corentin Dancette, Matthieu Cord, Mustafa Shukor","submitted_at":"2023-10-01T12:02:59Z","abstract_excerpt":"Following the success of Large Language Models (LLMs), Large Multimodal Models (LMMs), such as the Flamingo model and its subsequent competitors, have started to emerge as natural steps towards generalist agents. However, interacting with recent LMMs reveals major limitations that are hardly captured by the current evaluation benchmarks. Indeed, task performances (e.g., VQA accuracy) alone do not provide enough clues to understand their real capabilities, limitations, and to which extent such models are aligned to human expectations. To refine our understanding of those flaws, we deviate from "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.00647","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.00647/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:36:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/E59dpHyoKngoKMN61wZC1MlFJn4S5jANmPfHG8IVEH4xiKWvwpaIBqdPD7CJtDZOI6XjezWsxaM1MhLXzqIAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T19:27:18.366394Z"},"content_sha256":"47c3018afbb71047f12dba2e21e99e4d8c8618f7e3ca6d10d15ef063251efafb","schema_version":"1.0","event_id":"sha256:47c3018afbb71047f12dba2e21e99e4d8c8618f7e3ca6d10d15ef063251efafb"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/bundle.json","state_url":"https://pith.science/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T19:27:18Z","links":{"resolver":"https://pith.science/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B","bundle":"https://pith.science/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/bundle.json","state":"https://pith.science/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LSR2E3ISP5LAY2TT3QBKSLWU6B/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:LSR2E3ISP5LAY2TT3QBKSLWU6B","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"cad2663cbcde5d3e26a840c0dcf54923f102fe07d54a0b8d917f7f801680ac61","cross_cats_sorted":["cs.MM"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-10-01T12:02:59Z","title_canon_sha256":"48b2a39392920bbc837720e26a003ed2b3f3fe2af5f2ec94d191ee3530644d98"},"schema_version":"1.0","source":{"id":"2310.00647","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.00647","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"arxiv_version","alias_value":"2310.00647v2","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.00647","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_12","alias_value":"LSR2E3ISP5LA","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_16","alias_value":"LSR2E3ISP5LAY2TT","created_at":"2026-07-05T07:36:03Z"},{"alias_kind":"pith_short_8","alias_value":"LSR2E3IS","created_at":"2026-07-05T07:36:03Z"}],"graph_snapshots":[{"event_id":"sha256:47c3018afbb71047f12dba2e21e99e4d8c8618f7e3ca6d10d15ef063251efafb","target":"graph","created_at":"2026-07-05T07:36:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.00647/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Following the success of Large Language Models (LLMs), Large Multimodal Models (LMMs), such as the Flamingo model and its subsequent competitors, have started to emerge as natural steps towards generalist agents. However, interacting with recent LMMs reveals major limitations that are hardly captured by the current evaluation benchmarks. Indeed, task performances (e.g., VQA accuracy) alone do not provide enough clues to understand their real capabilities, limitations, and to which extent such models are aligned to human expectations. To refine our understanding of those flaws, we deviate from ","authors_text":"Alexandre Rame, Corentin Dancette, Matthieu Cord, Mustafa Shukor","cross_cats":["cs.MM"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-10-01T12:02:59Z","title":"Beyond Task Performance: Evaluating and Reducing the Flaws of Large Multimodal Models with In-Context Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.00647","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fd443f24fce514c207b44e79a5213c38cc22b22ddd0362fab26694a1209ab122","target":"record","created_at":"2026-07-05T07:36:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"cad2663cbcde5d3e26a840c0dcf54923f102fe07d54a0b8d917f7f801680ac61","cross_cats_sorted":["cs.MM"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-10-01T12:02:59Z","title_canon_sha256":"48b2a39392920bbc837720e26a003ed2b3f3fe2af5f2ec94d191ee3530644d98"},"schema_version":"1.0","source":{"id":"2310.00647","kind":"arxiv","version":2}},"canonical_sha256":"5ca3a26d127f560c6a73dc02a92ed4f07a603ad38e6558bebeca478b74da4e22","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5ca3a26d127f560c6a73dc02a92ed4f07a603ad38e6558bebeca478b74da4e22","first_computed_at":"2026-07-05T07:36:03.390974Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:36:03.390974Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"TURCEIThVu8ZjHPnmNl24bbGRfuDRuynOPdHBTG6+Z5+b1Fj8mUASA6g/I3H4Ap75FKHbmoUXH95d64d6qi+Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T07:36:03.391476Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.00647","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fd443f24fce514c207b44e79a5213c38cc22b22ddd0362fab26694a1209ab122","sha256:47c3018afbb71047f12dba2e21e99e4d8c8618f7e3ca6d10d15ef063251efafb"],"state_sha256":"1c1b18befe98e5c7948f51bbf5d1bce0921d024a3d43a0ebde442d777297211f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"daYHdMn803RDUpEKZ4L8xzwnhCGEYFSeyR1vzTuhdwR9RFIIAEXB0Z71oJLhbufL3I0QLyHOf5HwFVafl8GUCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T19:27:18.371680Z","bundle_sha256":"1e1db5a8f4e035a5d91ab022d56c482dd7f73ed5a7f5a8dc30a99d674eb7648d"}}