{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:JI4JF7MPL42LLBBHIIEEFDSWV4","short_pith_number":"pith:JI4JF7MP","canonical_record":{"source":{"id":"2411.10950","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-17T03:32:50Z","cross_cats_sorted":[],"title_canon_sha256":"83dfb4498124283dcb14e345c948c7e146764845467160690ac33daec0178482","abstract_canon_sha256":"e2a112cff15cd5e64c04d999298e7942440a7f21c0b2cff5516f61a40890e67b"},"schema_version":"1.0"},"canonical_sha256":"4a3892fd8f5f34b584274208428e56af37283a1243e10100dd73ddc259f0196c","source":{"kind":"arxiv","id":"2411.10950","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.10950","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"arxiv_version","alias_value":"2411.10950v2","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.10950","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_12","alias_value":"JI4JF7MPL42L","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_16","alias_value":"JI4JF7MPL42LLBBH","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_8","alias_value":"JI4JF7MP","created_at":"2026-07-05T09:59:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:JI4JF7MPL42LLBBHIIEEFDSWV4","target":"record","payload":{"canonical_record":{"source":{"id":"2411.10950","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-17T03:32:50Z","cross_cats_sorted":[],"title_canon_sha256":"83dfb4498124283dcb14e345c948c7e146764845467160690ac33daec0178482","abstract_canon_sha256":"e2a112cff15cd5e64c04d999298e7942440a7f21c0b2cff5516f61a40890e67b"},"schema_version":"1.0"},"canonical_sha256":"4a3892fd8f5f34b584274208428e56af37283a1243e10100dd73ddc259f0196c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:59:55.391057Z","signature_b64":"yDwks4HSGJua4iZug5/rJYttg66qOGzrWseKAmMbRzYZ1Qs48zYFDszvbdgdT5wzqLi7GjQII1HhK0BkrHAQCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4a3892fd8f5f34b584274208428e56af37283a1243e10100dd73ddc259f0196c","last_reissued_at":"2026-07-05T09:59:55.390594Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:59:55.390594Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.10950","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N1GBZziSeAEy0aLjxpw1trAurB6mV59DHu5tVrD5DR6Ctqbf5TXNlxjVB2ytm5M/VX6LrE8JK0UIB7pjwfKCBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T14:20:09.811982Z"},"content_sha256":"7bc4df9b2ac11cfe22a0d901466943217065d7697808a98d90894b8f5620463a","schema_version":"1.0","event_id":"sha256:7bc4df9b2ac11cfe22a0d901466943217065d7697808a98d90894b8f5620463a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:JI4JF7MPL42LLBBHIIEEFDSWV4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Understanding Multimodal LLMs: the Mechanistic Interpretability of Llava in Visual Question Answering","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Sophia Ananiadou, Zeping Yu","submitted_at":"2024-11-17T03:32:50Z","abstract_excerpt":"Understanding the mechanisms behind Large Language Models (LLMs) is crucial for designing improved models and strategies. While recent studies have yielded valuable insights into the mechanisms of textual LLMs, the mechanisms of Multi-modal Large Language Models (MLLMs) remain underexplored. In this paper, we apply mechanistic interpretability methods to analyze the visual question answering (VQA) mechanisms in the first MLLM, Llava. We compare the mechanisms between VQA and textual QA (TQA) in color answering tasks and find that: a) VQA exhibits a mechanism similar to the in-context learning "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.10950","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.10950/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qLZFcHJTK8kOl/5r1CRkSEUTuwXSa8Iv3EvdM/Oj/xWcjMNOtfNc2guBQJcLXkGJ+AHu8rGxARuIcdC+9GWLAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T14:20:09.812488Z"},"content_sha256":"ef1aa5f43b9a901b9a40c6fc73202f93430b8668a79ce54b9138e187d7ec12df","schema_version":"1.0","event_id":"sha256:ef1aa5f43b9a901b9a40c6fc73202f93430b8668a79ce54b9138e187d7ec12df"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/bundle.json","state_url":"https://pith.science/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T14:20:09Z","links":{"resolver":"https://pith.science/pith/JI4JF7MPL42LLBBHIIEEFDSWV4","bundle":"https://pith.science/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/bundle.json","state":"https://pith.science/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JI4JF7MPL42LLBBHIIEEFDSWV4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:JI4JF7MPL42LLBBHIIEEFDSWV4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e2a112cff15cd5e64c04d999298e7942440a7f21c0b2cff5516f61a40890e67b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-17T03:32:50Z","title_canon_sha256":"83dfb4498124283dcb14e345c948c7e146764845467160690ac33daec0178482"},"schema_version":"1.0","source":{"id":"2411.10950","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.10950","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"arxiv_version","alias_value":"2411.10950v2","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.10950","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_12","alias_value":"JI4JF7MPL42L","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_16","alias_value":"JI4JF7MPL42LLBBH","created_at":"2026-07-05T09:59:55Z"},{"alias_kind":"pith_short_8","alias_value":"JI4JF7MP","created_at":"2026-07-05T09:59:55Z"}],"graph_snapshots":[{"event_id":"sha256:ef1aa5f43b9a901b9a40c6fc73202f93430b8668a79ce54b9138e187d7ec12df","target":"graph","created_at":"2026-07-05T09:59:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.10950/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Understanding the mechanisms behind Large Language Models (LLMs) is crucial for designing improved models and strategies. While recent studies have yielded valuable insights into the mechanisms of textual LLMs, the mechanisms of Multi-modal Large Language Models (MLLMs) remain underexplored. In this paper, we apply mechanistic interpretability methods to analyze the visual question answering (VQA) mechanisms in the first MLLM, Llava. We compare the mechanisms between VQA and textual QA (TQA) in color answering tasks and find that: a) VQA exhibits a mechanism similar to the in-context learning ","authors_text":"Sophia Ananiadou, Zeping Yu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-17T03:32:50Z","title":"Understanding Multimodal LLMs: the Mechanistic Interpretability of Llava in Visual Question Answering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.10950","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7bc4df9b2ac11cfe22a0d901466943217065d7697808a98d90894b8f5620463a","target":"record","created_at":"2026-07-05T09:59:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e2a112cff15cd5e64c04d999298e7942440a7f21c0b2cff5516f61a40890e67b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-17T03:32:50Z","title_canon_sha256":"83dfb4498124283dcb14e345c948c7e146764845467160690ac33daec0178482"},"schema_version":"1.0","source":{"id":"2411.10950","kind":"arxiv","version":2}},"canonical_sha256":"4a3892fd8f5f34b584274208428e56af37283a1243e10100dd73ddc259f0196c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4a3892fd8f5f34b584274208428e56af37283a1243e10100dd73ddc259f0196c","first_computed_at":"2026-07-05T09:59:55.390594Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:59:55.390594Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yDwks4HSGJua4iZug5/rJYttg66qOGzrWseKAmMbRzYZ1Qs48zYFDszvbdgdT5wzqLi7GjQII1HhK0BkrHAQCw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:59:55.391057Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.10950","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7bc4df9b2ac11cfe22a0d901466943217065d7697808a98d90894b8f5620463a","sha256:ef1aa5f43b9a901b9a40c6fc73202f93430b8668a79ce54b9138e187d7ec12df"],"state_sha256":"034d7f3ed7ad827eed9874e0b3779d34cac727da922121b99fa1473f359f3d6f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EMZl4nb2FnfDewZqRvy3po7gUcoiOHURf70vHVbmX/9n8mFyxxWmhHfMZeEg7HvBHuLGzvkH+Z4bniIsg24WBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T14:20:09.826295Z","bundle_sha256":"359255c286a22bf2ea03d6922de5cb3e665937085ef609720299409703c8b027"}}