{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:6K6JTUR7YTKNZHLQ6TRTHHO5L6","short_pith_number":"pith:6K6JTUR7","canonical_record":{"source":{"id":"2605.13711","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-13T15:58:42Z","cross_cats_sorted":[],"title_canon_sha256":"f2ff510c0a7ffad21eae2068bb6374af48ce073625e9de0fb25724e0e9a7906b","abstract_canon_sha256":"b4759013a533d5e08aa327f6433c5196f6492bb84bbdae2d819cd17f77f3d5a8"},"schema_version":"1.0"},"canonical_sha256":"f2bc99d23fc4d4dc9d70f4e3339ddd5fa0652a16777fa4bdff0b8624ab8da476","source":{"kind":"arxiv","id":"2605.13711","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.13711","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"arxiv_version","alias_value":"2605.13711v1","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.13711","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"pith_short_12","alias_value":"6K6JTUR7YTKN","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"6K6JTUR7YTKNZHLQ","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"6K6JTUR7","created_at":"2026-05-18T12:33:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:6K6JTUR7YTKNZHLQ6TRTHHO5L6","target":"record","payload":{"canonical_record":{"source":{"id":"2605.13711","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-13T15:58:42Z","cross_cats_sorted":[],"title_canon_sha256":"f2ff510c0a7ffad21eae2068bb6374af48ce073625e9de0fb25724e0e9a7906b","abstract_canon_sha256":"b4759013a533d5e08aa327f6433c5196f6492bb84bbdae2d819cd17f77f3d5a8"},"schema_version":"1.0"},"canonical_sha256":"f2bc99d23fc4d4dc9d70f4e3339ddd5fa0652a16777fa4bdff0b8624ab8da476","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T02:44:16.752110Z","signature_b64":"EntVni66oOIhdYXQq6rYvlS9ieTk45kqgOXEVZ1gZ3a/vIT86IPD/8C4QS8BL5aohO3ZltJbCPaQrYbzc7nHDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f2bc99d23fc4d4dc9d70f4e3339ddd5fa0652a16777fa4bdff0b8624ab8da476","last_reissued_at":"2026-05-18T02:44:16.751580Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T02:44:16.751580Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2605.13711","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T02:44:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fODUTUZmTA4ocsAI1a57dSEZUV+hutHB0LmHmc6wsefcLOk2q0f0SpUVBxc23TRzhMFoBpKa4hYP5kZi4Nn4Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T11:52:49.686811Z"},"content_sha256":"0ed6c7d249e46270642d473bab57a022ac87b45232628c2dff3e72e62a3bb739","schema_version":"1.0","event_id":"sha256:0ed6c7d249e46270642d473bab57a022ac87b45232628c2dff3e72e62a3bb739"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:6K6JTUR7YTKNZHLQ6TRTHHO5L6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MILM: Large Language Models for Multimodal Irregular Time Series with Informative Sampling","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning.","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Hsing-Huan Chung, Joydeep Ghosh, Shijun Li, Suchi Saria, Xing Han, Yoav Wald","submitted_at":"2026-05-13T15:58:42Z","abstract_excerpt":"Multimodal irregular time series (MITS) consist of asynchronous and irregularly sampled observations from heterogeneous numerical and textual channels. In healthcare, for example, patients' electronic health records (EHR) include irregular lab measurements and clinical notes. The irregular timing and channel patterns of observations carry predictive signal alongside the numerical values and textual content. LLMs are natural candidates for processing such heterogeneous data, given their extensive pretrained knowledge spanning textual and numerical domains. We introduce MILM (Multimodal Irregula"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Our two-stage model (MILM-2S) and its single-stage counterpart (MILM-Direct) achieve the best and second-best average performance on multiple EHR datasets. Further value redaction evaluations confirm that sampling patterns carry predictive signal and that MILM-2S learns to exploit them.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That representing MITS as time-ordered triplets in XML format allows pretrained LLMs to effectively capture and exploit irregular sampling patterns alongside numerical and textual content without significant loss of temporal structure.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"MILM fine-tunes LLMs on XML-encoded multimodal irregular time series via a two-stage process that exploits informative sampling patterns to achieve top performance on EHR classification datasets.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"7e6f4d8c53ffb0e52756df5119676ca400b5475fbee938d1772ac3019cdc20b4"},"source":{"id":"2605.13711","kind":"arxiv","version":1},"verdict":{"id":"c38a7f92-8394-4fb0-8897-790b3b563932","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-14T20:13:44.741809Z","strongest_claim":"Our two-stage model (MILM-2S) and its single-stage counterpart (MILM-Direct) achieve the best and second-best average performance on multiple EHR datasets. Further value redaction evaluations confirm that sampling patterns carry predictive signal and that MILM-2S learns to exploit them.","one_line_summary":"MILM fine-tunes LLMs on XML-encoded multimodal irregular time series via a two-stage process that exploits informative sampling patterns to achieve top performance on EHR classification datasets.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That representing MITS as time-ordered triplets in XML format allows pretrained LLMs to effectively capture and exploit irregular sampling patterns alongside numerical and textual content without significant loss of temporal structure.","pith_extraction_headline":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning."},"references":{"count":84,"sample":[{"doi":"","year":2025,"title":"Time-IMM: A dataset and benchmark for irregular multimodal multivariate time series","work_id":"1bcfb18f-685c-4e0c-8cef-c16ca0ef43ea","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2023,"title":"Improving medical predictions by irregular multimodal electronic health records modeling","work_id":"256d2789-487b-42be-a1e4-dd0332e6ba9c","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2025,"title":"Multimodal language models for financial forecasting from interleaved sequences of text and time series","work_id":"99746b6c-7e60-4c05-8974-5675e71b4188","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2024,"title":"A survey of aiops for failure management in the era of large language models","work_id":"545ee6b5-d225-4418-9a73-6851a0f8d129","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2023,"title":"Mimic-iv, a freely accessible electronic health record dataset.Scientific data, 10(1):1","work_id":"db1513ca-6a70-47be-ae26-57338b07a878","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":84,"snapshot_sha256":"ae919ebb27300e78aa4a0068a9fc733fecfdb5778b86e4bb16a4c9e1e38ce50d","internal_anchors":7},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"c38a7f92-8394-4fb0-8897-790b3b563932"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T02:44:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Pn4/CpmwMb5XUFzrTDOY5p2yYcosMM1K66BQ9m270WvLb1nQs1TW6xEtji3/FkkPE3mndhQWiklKulhqKX7dCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T11:52:49.687342Z"},"content_sha256":"dce83436d07b8b30a3214b59695d009a453f42de8a3beede41dfb05fee6145ec","schema_version":"1.0","event_id":"sha256:dce83436d07b8b30a3214b59695d009a453f42de8a3beede41dfb05fee6145ec"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/bundle.json","state_url":"https://pith.science/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-01T11:52:49Z","links":{"resolver":"https://pith.science/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6","bundle":"https://pith.science/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/bundle.json","state":"https://pith.science/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6K6JTUR7YTKNZHLQ6TRTHHO5L6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:6K6JTUR7YTKNZHLQ6TRTHHO5L6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b4759013a533d5e08aa327f6433c5196f6492bb84bbdae2d819cd17f77f3d5a8","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-13T15:58:42Z","title_canon_sha256":"f2ff510c0a7ffad21eae2068bb6374af48ce073625e9de0fb25724e0e9a7906b"},"schema_version":"1.0","source":{"id":"2605.13711","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.13711","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"arxiv_version","alias_value":"2605.13711v1","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.13711","created_at":"2026-05-18T02:44:16Z"},{"alias_kind":"pith_short_12","alias_value":"6K6JTUR7YTKN","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"6K6JTUR7YTKNZHLQ","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"6K6JTUR7","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:dce83436d07b8b30a3214b59695d009a453f42de8a3beede41dfb05fee6145ec","target":"graph","created_at":"2026-05-18T02:44:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Our two-stage model (MILM-2S) and its single-stage counterpart (MILM-Direct) achieve the best and second-best average performance on multiple EHR datasets. Further value redaction evaluations confirm that sampling patterns carry predictive signal and that MILM-2S learns to exploit them."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That representing MITS as time-ordered triplets in XML format allows pretrained LLMs to effectively capture and exploit irregular sampling patterns alongside numerical and textual content without significant loss of temporal structure."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"MILM fine-tunes LLMs on XML-encoded multimodal irregular time series via a two-stage process that exploits informative sampling patterns to achieve top performance on EHR classification datasets."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning."}],"snapshot_sha256":"7e6f4d8c53ffb0e52756df5119676ca400b5475fbee938d1772ac3019cdc20b4"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Multimodal irregular time series (MITS) consist of asynchronous and irregularly sampled observations from heterogeneous numerical and textual channels. In healthcare, for example, patients' electronic health records (EHR) include irregular lab measurements and clinical notes. The irregular timing and channel patterns of observations carry predictive signal alongside the numerical values and textual content. LLMs are natural candidates for processing such heterogeneous data, given their extensive pretrained knowledge spanning textual and numerical domains. We introduce MILM (Multimodal Irregula","authors_text":"Hsing-Huan Chung, Joydeep Ghosh, Shijun Li, Suchi Saria, Xing Han, Yoav Wald","cross_cats":[],"headline":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-13T15:58:42Z","title":"MILM: Large Language Models for Multimodal Irregular Time Series with Informative Sampling"},"references":{"count":84,"internal_anchors":7,"resolved_work":84,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Time-IMM: A dataset and benchmark for irregular multimodal multivariate time series","work_id":"1bcfb18f-685c-4e0c-8cef-c16ca0ef43ea","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Improving medical predictions by irregular multimodal electronic health records modeling","work_id":"256d2789-487b-42be-a1e4-dd0332e6ba9c","year":2023},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Multimodal language models for financial forecasting from interleaved sequences of text and time series","work_id":"99746b6c-7e60-4c05-8974-5675e71b4188","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"A survey of aiops for failure management in the era of large language models","work_id":"545ee6b5-d225-4418-9a73-6851a0f8d129","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Mimic-iv, a freely accessible electronic health record dataset.Scientific data, 10(1):1","work_id":"db1513ca-6a70-47be-ae26-57338b07a878","year":2023}],"snapshot_sha256":"ae919ebb27300e78aa4a0068a9fc733fecfdb5778b86e4bb16a4c9e1e38ce50d"},"source":{"id":"2605.13711","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-14T20:13:44.741809Z","id":"c38a7f92-8394-4fb0-8897-790b3b563932","model_set":{"reader":"grok-4.3"},"one_line_summary":"MILM fine-tunes LLMs on XML-encoded multimodal irregular time series via a two-stage process that exploits informative sampling patterns to achieve top performance on EHR classification datasets.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Large language models can exploit irregular sampling patterns in multimodal time series by representing them as XML triplets and using two-stage fine-tuning.","strongest_claim":"Our two-stage model (MILM-2S) and its single-stage counterpart (MILM-Direct) achieve the best and second-best average performance on multiple EHR datasets. Further value redaction evaluations confirm that sampling patterns carry predictive signal and that MILM-2S learns to exploit them.","weakest_assumption":"That representing MITS as time-ordered triplets in XML format allows pretrained LLMs to effectively capture and exploit irregular sampling patterns alongside numerical and textual content without significant loss of temporal structure."}},"verdict_id":"c38a7f92-8394-4fb0-8897-790b3b563932"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0ed6c7d249e46270642d473bab57a022ac87b45232628c2dff3e72e62a3bb739","target":"record","created_at":"2026-05-18T02:44:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b4759013a533d5e08aa327f6433c5196f6492bb84bbdae2d819cd17f77f3d5a8","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-13T15:58:42Z","title_canon_sha256":"f2ff510c0a7ffad21eae2068bb6374af48ce073625e9de0fb25724e0e9a7906b"},"schema_version":"1.0","source":{"id":"2605.13711","kind":"arxiv","version":1}},"canonical_sha256":"f2bc99d23fc4d4dc9d70f4e3339ddd5fa0652a16777fa4bdff0b8624ab8da476","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f2bc99d23fc4d4dc9d70f4e3339ddd5fa0652a16777fa4bdff0b8624ab8da476","first_computed_at":"2026-05-18T02:44:16.751580Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T02:44:16.751580Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"EntVni66oOIhdYXQq6rYvlS9ieTk45kqgOXEVZ1gZ3a/vIT86IPD/8C4QS8BL5aohO3ZltJbCPaQrYbzc7nHDA==","signature_status":"signed_v1","signed_at":"2026-05-18T02:44:16.752110Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.13711","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0ed6c7d249e46270642d473bab57a022ac87b45232628c2dff3e72e62a3bb739","sha256:dce83436d07b8b30a3214b59695d009a453f42de8a3beede41dfb05fee6145ec"],"state_sha256":"1ab987b81c1c8b97622e3e02d62b5af25c3e6417061f2d31f77e5c2903032e3f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eBFWVDcl3bDbPjREP5sYXmBnZTAkoO2bCu1cwE/QotlRKIW0QVKabEEmJHornzjmxd5SNoVmsg4pDVNy1uxXDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-01T11:52:49.690636Z","bundle_sha256":"6e712b8c56dad14e8329fb5ddba34403e175e3b32d013221e42789ad6bbe357a"}}