{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:OAT6C4D65T7R6VSVPFL75U5NCZ","short_pith_number":"pith:OAT6C4D6","canonical_record":{"source":{"id":"2402.08017","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-02-12T19:27:26Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"934ea480631f85ceb37a5a8c07789055125ac3d1dc501b2a2db3ded3e9729975","abstract_canon_sha256":"7272adbe07e38fe0f629b6d1908d98a296a3477aaba30ee3bf9f919e647db046"},"schema_version":"1.0"},"canonical_sha256":"7027e1707eecff1f56557957fed3ad16669a44dd1a753e5efad982a19f650ec8","source":{"kind":"arxiv","id":"2402.08017","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.08017","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"arxiv_version","alias_value":"2402.08017v2","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.08017","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_12","alias_value":"OAT6C4D65T7R","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_16","alias_value":"OAT6C4D65T7R6VSV","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_8","alias_value":"OAT6C4D6","created_at":"2026-07-05T08:26:19Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:OAT6C4D65T7R6VSVPFL75U5NCZ","target":"record","payload":{"canonical_record":{"source":{"id":"2402.08017","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-02-12T19:27:26Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"934ea480631f85ceb37a5a8c07789055125ac3d1dc501b2a2db3ded3e9729975","abstract_canon_sha256":"7272adbe07e38fe0f629b6d1908d98a296a3477aaba30ee3bf9f919e647db046"},"schema_version":"1.0"},"canonical_sha256":"7027e1707eecff1f56557957fed3ad16669a44dd1a753e5efad982a19f650ec8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:26:19.368026Z","signature_b64":"XfRnaPlnMVXORKZ43V883f8I+olEsLcK69TgYxVFVhxWuRPEfcVA34l/6WpPZyP+IJGQJ0rPMGwVFtsXYR+sDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7027e1707eecff1f56557957fed3ad16669a44dd1a753e5efad982a19f650ec8","last_reissued_at":"2026-07-05T08:26:19.367523Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:26:19.367523Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.08017","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:26:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zfgDZ9mqSJDVFYC8k1JxlCvAL/t6m3SyGWwsVfxh+I7p4c651XQj0GdHaE+EwnYU09xTDCZDTk4OABRrv+85Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T20:59:55.875634Z"},"content_sha256":"8479ffd4ea77d54bbf897179301c02bac4a607c137c3ad015189ac04a4bbbf5d","schema_version":"1.0","event_id":"sha256:8479ffd4ea77d54bbf897179301c02bac4a607c137c3ad015189ac04a4bbbf5d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:OAT6C4D65T7R6VSVPFL75U5NCZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Lumos : Empowering Multimodal LLMs with Scene Text Recognition","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.CV","authors_text":"Abhay Harpale, Ankit Ramchandani, Anuj Kumar, Ashish Shenoy, Debojeet Chatterjee, Di Xu, Longfang Zhao, Mohsen Moslehpour, Pierce Chuang, Shicong Zhao, Srihari Jayakumar, Vikas Bhardwaj, Xin Luna Dong, Yichao Lu","submitted_at":"2024-02-12T19:27:26Z","abstract_excerpt":"We introduce Lumos, the first end-to-end multimodal question-answering system with text understanding capabilities. At the core of Lumos is a Scene Text Recognition (STR) component that extracts text from first person point-of-view images, the output of which is used to augment input to a Multimodal Large Language Model (MM-LLM). While building Lumos, we encountered numerous challenges related to STR quality, overall latency, and model inference. In this paper, we delve into those challenges, and discuss the system architecture, design choices, and modeling techniques employed to overcome thes"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.08017","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.08017/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:26:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PTL+eVAleS1YHCQiAju0H5/haBKvkzxPGR4N7JElTNBqmLvHwFj7ky68fEDeaSZhzWrIfJySNMI/IbLsYXbUAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T20:59:55.876021Z"},"content_sha256":"eba825961b416fc6f782caf7dfbd620b1d9cdd588609b8518038ea585ee9de57","schema_version":"1.0","event_id":"sha256:eba825961b416fc6f782caf7dfbd620b1d9cdd588609b8518038ea585ee9de57"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/bundle.json","state_url":"https://pith.science/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T20:59:55Z","links":{"resolver":"https://pith.science/pith/OAT6C4D65T7R6VSVPFL75U5NCZ","bundle":"https://pith.science/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/bundle.json","state":"https://pith.science/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OAT6C4D65T7R6VSVPFL75U5NCZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:OAT6C4D65T7R6VSVPFL75U5NCZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7272adbe07e38fe0f629b6d1908d98a296a3477aaba30ee3bf9f919e647db046","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-02-12T19:27:26Z","title_canon_sha256":"934ea480631f85ceb37a5a8c07789055125ac3d1dc501b2a2db3ded3e9729975"},"schema_version":"1.0","source":{"id":"2402.08017","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.08017","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"arxiv_version","alias_value":"2402.08017v2","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.08017","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_12","alias_value":"OAT6C4D65T7R","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_16","alias_value":"OAT6C4D65T7R6VSV","created_at":"2026-07-05T08:26:19Z"},{"alias_kind":"pith_short_8","alias_value":"OAT6C4D6","created_at":"2026-07-05T08:26:19Z"}],"graph_snapshots":[{"event_id":"sha256:eba825961b416fc6f782caf7dfbd620b1d9cdd588609b8518038ea585ee9de57","target":"graph","created_at":"2026-07-05T08:26:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.08017/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We introduce Lumos, the first end-to-end multimodal question-answering system with text understanding capabilities. At the core of Lumos is a Scene Text Recognition (STR) component that extracts text from first person point-of-view images, the output of which is used to augment input to a Multimodal Large Language Model (MM-LLM). While building Lumos, we encountered numerous challenges related to STR quality, overall latency, and model inference. In this paper, we delve into those challenges, and discuss the system architecture, design choices, and modeling techniques employed to overcome thes","authors_text":"Abhay Harpale, Ankit Ramchandani, Anuj Kumar, Ashish Shenoy, Debojeet Chatterjee, Di Xu, Longfang Zhao, Mohsen Moslehpour, Pierce Chuang, Shicong Zhao, Srihari Jayakumar, Vikas Bhardwaj, Xin Luna Dong, Yichao Lu","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-02-12T19:27:26Z","title":"Lumos : Empowering Multimodal LLMs with Scene Text Recognition"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.08017","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8479ffd4ea77d54bbf897179301c02bac4a607c137c3ad015189ac04a4bbbf5d","target":"record","created_at":"2026-07-05T08:26:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7272adbe07e38fe0f629b6d1908d98a296a3477aaba30ee3bf9f919e647db046","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-02-12T19:27:26Z","title_canon_sha256":"934ea480631f85ceb37a5a8c07789055125ac3d1dc501b2a2db3ded3e9729975"},"schema_version":"1.0","source":{"id":"2402.08017","kind":"arxiv","version":2}},"canonical_sha256":"7027e1707eecff1f56557957fed3ad16669a44dd1a753e5efad982a19f650ec8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7027e1707eecff1f56557957fed3ad16669a44dd1a753e5efad982a19f650ec8","first_computed_at":"2026-07-05T08:26:19.367523Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:26:19.367523Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XfRnaPlnMVXORKZ43V883f8I+olEsLcK69TgYxVFVhxWuRPEfcVA34l/6WpPZyP+IJGQJ0rPMGwVFtsXYR+sDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:26:19.368026Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.08017","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8479ffd4ea77d54bbf897179301c02bac4a607c137c3ad015189ac04a4bbbf5d","sha256:eba825961b416fc6f782caf7dfbd620b1d9cdd588609b8518038ea585ee9de57"],"state_sha256":"fba837b304db30abfb4ee5ca7a158ee89f49ec711cd590c6f0780f92c759251c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iQ3f+8krqG+n+me2K1yGrrogTgzpzxVdcq0LI1rXbYzwvs+Etv16RS5KUNA1kjM0cRm/Eb11YtwCdv2jE1EOCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T20:59:55.878059Z","bundle_sha256":"27208f370c8c019a5e133dfa6f7a1131834c60a09494e7b316ee2d3422ca2026"}}