{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:LRMVZIQ4PYFXAYYZLNGLEZQUGT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"503a5d5ebe496c4c6f24c513eecce7c4434bffaebbd30fd8d393cbf674ec4041","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-14T07:35:58Z","title_canon_sha256":"11b8a06a8ccb4408cfc2bc36fdd84d56c8f412eb4767a63ea1f8f4168572c3aa"},"schema_version":"1.0","source":{"id":"2605.14497","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.14497","created_at":"2026-05-17T23:39:06Z"},{"alias_kind":"arxiv_version","alias_value":"2605.14497v1","created_at":"2026-05-17T23:39:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.14497","created_at":"2026-05-17T23:39:06Z"},{"alias_kind":"pith_short_12","alias_value":"LRMVZIQ4PYFX","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"LRMVZIQ4PYFXAYYZ","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"LRMVZIQ4","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:4f77dd571a0aecfedb3d94006ad0c26cf332c817c0c2e449a860e1040c816fea","target":"graph","created_at":"2026-05-17T23:39:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Our empirical results demonstrate that this approach consistently outperforms existing data replay methods across various datasets, eliminating the need for manual, context-specific adjustments while achieving superior stability and asymptotic performance."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The surrogate objective used inside the multi-armed bandit sufficiently approximates the true bi-level gradient so that the outer-level data-mixing decisions actually improve the final policy performance."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"ROAD formulates data mixing as a bi-level optimization problem solved via multi-armed bandit to adaptively balance offline priors and online updates in RL."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"ROAD frames data mixing in offline-to-online reinforcement learning as a bi-level optimization problem solved by a multi-armed bandit to automate replay ratios."}],"snapshot_sha256":"1c166bf7024dfe7cecaf7df41bf6802b5b89ee442914f45a6c5873b0f1a858cc"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Offline-to-online reinforcement learning harnesses the stability of offline pretraining and the flexibility of online fine-tuning. A key challenge lies in the non-stationary distribution shift between offline datasets and the evolving online policy. Common approaches often rely on static mixing ratios or heuristic-based replay strategies, which lack adaptability to different environments and varying training dynamics, resulting in suboptimal tradeoff between stability and asymptotic performance. In this work, we propose Reinforcement Learning with Optimized Adaptive Data-mixing (ROAD), a dynam","authors_text":"(2) Ant Group, China, China), Jian Liu (2), Letian Yang (1), Shanghai, Shuai Li (1) ((1) Shanghai Jiao Tong University, Weiqiang Wang (2), Xu Liu (1), Yiqiang Lu (2)","cross_cats":["cs.AI"],"headline":"ROAD frames data mixing in offline-to-online reinforcement learning as a bi-level optimization problem solved by a multi-armed bandit to automate replay ratios.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-14T07:35:58Z","title":"ROAD: Adaptive Data Mixing for Offline-to-Online Reinforcement Learning via Bi-Level Optimization"},"references":{"count":41,"internal_anchors":4,"resolved_work":41,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Efficient online reinforcement learning with offline data","work_id":"5627947c-d71f-483d-9d92-118dd0a35695","year":2023},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"MOORL: A frame- work for integrating offline-online reinforcement learning","work_id":"a1a8774d-e7ee-44c7-9c9a-ec40d70d6fce","year":2025},{"cited_arxiv_id":"2004.07219","doi":"","is_internal_anchor":true,"ref_index":3,"title":"D4RL: Datasets for Deep Data-Driven Reinforcement Learning","work_id":"47082e4e-a4a5-418b-bf4f-4667355065fc","year":2020},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learn- ing with a stochastic actor","work_id":"7455be7a-76a8-4f4d-b473-4eb7aeb56a6a","year":2018},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Modem: Accelerating visual model-based reinforcement learning with demonstrations","work_id":"946033c5-169f-4931-bdcc-c4f2a4e57c1e","year":2023}],"snapshot_sha256":"b8249927b62f2e45c742b23ccd08e7a3b7267d5fe8be72cee7e3a3a050a4dd8b"},"source":{"id":"2605.14497","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-15T01:29:05.909847Z","id":"89184cba-a250-4c66-869b-e789146dfbf8","model_set":{"reader":"grok-4.3"},"one_line_summary":"ROAD formulates data mixing as a bi-level optimization problem solved via multi-armed bandit to adaptively balance offline priors and online updates in RL.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"ROAD frames data mixing in offline-to-online reinforcement learning as a bi-level optimization problem solved by a multi-armed bandit to automate replay ratios.","strongest_claim":"Our empirical results demonstrate that this approach consistently outperforms existing data replay methods across various datasets, eliminating the need for manual, context-specific adjustments while achieving superior stability and asymptotic performance.","weakest_assumption":"The surrogate objective used inside the multi-armed bandit sufficiently approximates the true bi-level gradient so that the outer-level data-mixing decisions actually improve the final policy performance."}},"verdict_id":"89184cba-a250-4c66-869b-e789146dfbf8"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1941a995ec051213142093efd9c1864c81b843d0dd2409ab8819e5cb04e71807","target":"record","created_at":"2026-05-17T23:39:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"503a5d5ebe496c4c6f24c513eecce7c4434bffaebbd30fd8d393cbf674ec4041","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-14T07:35:58Z","title_canon_sha256":"11b8a06a8ccb4408cfc2bc36fdd84d56c8f412eb4767a63ea1f8f4168572c3aa"},"schema_version":"1.0","source":{"id":"2605.14497","kind":"arxiv","version":1}},"canonical_sha256":"5c595ca21c7e0b7063195b4cb2661434fe63434295697c0a11e39585a72f9109","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5c595ca21c7e0b7063195b4cb2661434fe63434295697c0a11e39585a72f9109","first_computed_at":"2026-05-17T23:39:06.361561Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:39:06.361561Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"E14WwEEV/+RI/+humVp75ekqvRU+rBkT0NwmYJOYiDe4Zyzg+JIM5qaQRaWUtzavezx9aHnsiesARr8uhgQbCQ==","signature_status":"signed_v1","signed_at":"2026-05-17T23:39:06.362249Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.14497","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1941a995ec051213142093efd9c1864c81b843d0dd2409ab8819e5cb04e71807","sha256:4f77dd571a0aecfedb3d94006ad0c26cf332c817c0c2e449a860e1040c816fea"],"state_sha256":"9a0f10afc5bb054d204bf45bbdb412c90a0f60a2bb8af13bea67a70b98ea8e57"}