{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:F6HUQNK4EUV5X6ONM6I3XEIYAI","short_pith_number":"pith:F6HUQNK4","canonical_record":{"source":{"id":"2311.17647","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-11-29T14:08:53Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"89fd98607bf18679b2821ebac5a51eb68042464e0965efcba752912acc784f67","abstract_canon_sha256":"87cd82c66ac17b8294995d35602ccba2e2e4aef3902f88416c86efe787ddd5fb"},"schema_version":"1.0"},"canonical_sha256":"2f8f48355c252bdbf9cd6791bb9118023cae5f79519aa1e0fb334b73fa1fd6d7","source":{"kind":"arxiv","id":"2311.17647","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.17647","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"arxiv_version","alias_value":"2311.17647v2","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.17647","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_12","alias_value":"F6HUQNK4EUV5","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_16","alias_value":"F6HUQNK4EUV5X6ON","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_8","alias_value":"F6HUQNK4","created_at":"2026-07-05T08:29:45Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:F6HUQNK4EUV5X6ONM6I3XEIYAI","target":"record","payload":{"canonical_record":{"source":{"id":"2311.17647","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-11-29T14:08:53Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"89fd98607bf18679b2821ebac5a51eb68042464e0965efcba752912acc784f67","abstract_canon_sha256":"87cd82c66ac17b8294995d35602ccba2e2e4aef3902f88416c86efe787ddd5fb"},"schema_version":"1.0"},"canonical_sha256":"2f8f48355c252bdbf9cd6791bb9118023cae5f79519aa1e0fb334b73fa1fd6d7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:29:45.344733Z","signature_b64":"9qOsUPkkaBs/8ClVpNqtWURbBPYpxH7XEloXlAJcH1S15mtxjbzxmCu7Big9Tyi03syQaqm5UOVoSIH8WiAsDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f8f48355c252bdbf9cd6791bb9118023cae5f79519aa1e0fb334b73fa1fd6d7","last_reissued_at":"2026-07-05T08:29:45.344239Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:29:45.344239Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.17647","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:29:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yxKW1EeP8913BEp7r7kD+nTeBU13iY1XtwuyAf0XxRBxVLqflICLniG4SDeRS2P+hoN0kNGUIp9HOsZJlBkLBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:20:13.264019Z"},"content_sha256":"f3f157c5ea2509225ac22850e1a9f7b3f9965e464f6cc8b9ceda93eacb768fbc","schema_version":"1.0","event_id":"sha256:f3f157c5ea2509225ac22850e1a9f7b3f9965e464f6cc8b9ceda93eacb768fbc"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:F6HUQNK4EUV5X6ONM6I3XEIYAI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Text as Images: Can Multimodal Large Language Models Follow Printed Instructions in Pixels?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.CV","authors_text":"Jianfeng Gao, William Yang Wang, Xiujun Li, Yejin Choi, Yujie Lu, Zhe Gan","submitted_at":"2023-11-29T14:08:53Z","abstract_excerpt":"Recent multimodal large language models (MLLMs) have shown promising instruction following capabilities on vision-language tasks. In this work, we introduce VISUAL MODALITY INSTRUCTION (VIM), and investigate how well multimodal models can understand textual instructions provided in pixels, despite not being explicitly trained on such data during pretraining or fine-tuning. We adapt VIM to eight benchmarks, including OKVQA, MM-Vet, MathVista, MMMU, and probe diverse MLLMs in both the text-modality instruction (TEM) setting and VIM setting. Notably, we observe a significant performance disparity"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.17647","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.17647/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:29:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WAlLglHm58f3L0EfoemNh4ENN+G3ayaFcWIiQtMdZOy7+9X4tMc2ghECsKkddAVzusUhoNgEN6laXl4oTKwEBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:20:13.264568Z"},"content_sha256":"18a14c8db5945cbced7d0797cbacf6c7cf3f1061ffdd5fb6f92076677256691c","schema_version":"1.0","event_id":"sha256:18a14c8db5945cbced7d0797cbacf6c7cf3f1061ffdd5fb6f92076677256691c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/bundle.json","state_url":"https://pith.science/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T12:20:13Z","links":{"resolver":"https://pith.science/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI","bundle":"https://pith.science/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/bundle.json","state":"https://pith.science/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F6HUQNK4EUV5X6ONM6I3XEIYAI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:F6HUQNK4EUV5X6ONM6I3XEIYAI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"87cd82c66ac17b8294995d35602ccba2e2e4aef3902f88416c86efe787ddd5fb","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-11-29T14:08:53Z","title_canon_sha256":"89fd98607bf18679b2821ebac5a51eb68042464e0965efcba752912acc784f67"},"schema_version":"1.0","source":{"id":"2311.17647","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.17647","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"arxiv_version","alias_value":"2311.17647v2","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.17647","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_12","alias_value":"F6HUQNK4EUV5","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_16","alias_value":"F6HUQNK4EUV5X6ON","created_at":"2026-07-05T08:29:45Z"},{"alias_kind":"pith_short_8","alias_value":"F6HUQNK4","created_at":"2026-07-05T08:29:45Z"}],"graph_snapshots":[{"event_id":"sha256:18a14c8db5945cbced7d0797cbacf6c7cf3f1061ffdd5fb6f92076677256691c","target":"graph","created_at":"2026-07-05T08:29:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.17647/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent multimodal large language models (MLLMs) have shown promising instruction following capabilities on vision-language tasks. In this work, we introduce VISUAL MODALITY INSTRUCTION (VIM), and investigate how well multimodal models can understand textual instructions provided in pixels, despite not being explicitly trained on such data during pretraining or fine-tuning. We adapt VIM to eight benchmarks, including OKVQA, MM-Vet, MathVista, MMMU, and probe diverse MLLMs in both the text-modality instruction (TEM) setting and VIM setting. Notably, we observe a significant performance disparity","authors_text":"Jianfeng Gao, William Yang Wang, Xiujun Li, Yejin Choi, Yujie Lu, Zhe Gan","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-11-29T14:08:53Z","title":"Text as Images: Can Multimodal Large Language Models Follow Printed Instructions in Pixels?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.17647","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f3f157c5ea2509225ac22850e1a9f7b3f9965e464f6cc8b9ceda93eacb768fbc","target":"record","created_at":"2026-07-05T08:29:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"87cd82c66ac17b8294995d35602ccba2e2e4aef3902f88416c86efe787ddd5fb","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-11-29T14:08:53Z","title_canon_sha256":"89fd98607bf18679b2821ebac5a51eb68042464e0965efcba752912acc784f67"},"schema_version":"1.0","source":{"id":"2311.17647","kind":"arxiv","version":2}},"canonical_sha256":"2f8f48355c252bdbf9cd6791bb9118023cae5f79519aa1e0fb334b73fa1fd6d7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2f8f48355c252bdbf9cd6791bb9118023cae5f79519aa1e0fb334b73fa1fd6d7","first_computed_at":"2026-07-05T08:29:45.344239Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:29:45.344239Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9qOsUPkkaBs/8ClVpNqtWURbBPYpxH7XEloXlAJcH1S15mtxjbzxmCu7Big9Tyi03syQaqm5UOVoSIH8WiAsDA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:29:45.344733Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.17647","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f3f157c5ea2509225ac22850e1a9f7b3f9965e464f6cc8b9ceda93eacb768fbc","sha256:18a14c8db5945cbced7d0797cbacf6c7cf3f1061ffdd5fb6f92076677256691c"],"state_sha256":"16f5b28f8ffdf7726956f292844ba741d6fc1cb24405358de56874265d57c260"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8qmuelH5JNuX9vULmz0OXOfqsXJ3nJLRXIAgEgj6SEVjqHrp8alua/wMVVy916oPzg5D7IGre2Wv9yGLJTdZAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T12:20:13.269484Z","bundle_sha256":"9258c8baf4855fb22497a653da466e63240c7dc11cc130faee562e34c8f88c70"}}