{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:XFRLHEB5KVWRYUIPE47OJV4RZU","short_pith_number":"pith:XFRLHEB5","schema_version":"1.0","canonical_sha256":"b962b3903d556d1c510f273ee4d791cd2c3ccea27190b0cdda0e8b2398e67cdb","source":{"kind":"arxiv","id":"2412.07762","version":3},"attestation_state":"computed","paper":{"title":"Efficient Online Reinforcement Learning Fine-Tuning Need Not Retain Offline Data","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Andy Peng, Aviral Kumar, Qiyang Li, Sergey Levine, Zhiyuan Zhou","submitted_at":"2024-12-10T18:57:12Z","abstract_excerpt":"The modern paradigm in machine learning involves pre-training on diverse data, followed by task-specific fine-tuning. In reinforcement learning (RL), this translates to learning via offline RL on a diverse historical dataset, followed by rapid online RL fine-tuning using interaction data. Most RL fine-tuning methods require continued training on offline data for stability and performance. However, this is undesirable because training on diverse offline data is slow and expensive for large datasets, and in principle, also limit the performance improvement possible because of constraints or pess"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.07762","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-12-10T18:57:12Z","cross_cats_sorted":[],"title_canon_sha256":"01f6188ae65f25c9bb7ef8309d5cd352d57c58e56e551cebca37dc39f33ef182","abstract_canon_sha256":"df614d70855dabc961cc5e71d300ea04b6a190951bd245ceb06c56126c238944"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:30:39.858436Z","signature_b64":"HTRYWArw7BGk3IQGSfsDBPYWF1AR9uXJs1h+6CJ6AiukH+iNqc4ADECeYtQBmPqPzGQFHrKltoS3by+8OkzOBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b962b3903d556d1c510f273ee4d791cd2c3ccea27190b0cdda0e8b2398e67cdb","last_reissued_at":"2026-07-05T11:30:39.857905Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:30:39.857905Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Efficient Online Reinforcement Learning Fine-Tuning Need Not Retain Offline Data","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Andy Peng, Aviral Kumar, Qiyang Li, Sergey Levine, Zhiyuan Zhou","submitted_at":"2024-12-10T18:57:12Z","abstract_excerpt":"The modern paradigm in machine learning involves pre-training on diverse data, followed by task-specific fine-tuning. In reinforcement learning (RL), this translates to learning via offline RL on a diverse historical dataset, followed by rapid online RL fine-tuning using interaction data. Most RL fine-tuning methods require continued training on offline data for stability and performance. However, this is undesirable because training on diverse offline data is slow and expensive for large datasets, and in principle, also limit the performance improvement possible because of constraints or pess"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.07762","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.07762/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.07762","created_at":"2026-07-05T11:30:39.857969+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.07762v3","created_at":"2026-07-05T11:30:39.857969+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.07762","created_at":"2026-07-05T11:30:39.857969+00:00"},{"alias_kind":"pith_short_12","alias_value":"XFRLHEB5KVWR","created_at":"2026-07-05T11:30:39.857969+00:00"},{"alias_kind":"pith_short_16","alias_value":"XFRLHEB5KVWRYUIP","created_at":"2026-07-05T11:30:39.857969+00:00"},{"alias_kind":"pith_short_8","alias_value":"XFRLHEB5","created_at":"2026-07-05T11:30:39.857969+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":11,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.14779","citing_title":"Peng's Q($\\lambda$) for Conservative Value Estimation in Offline Reinforcement Learning","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03065","citing_title":"OGPO: Sample Efficient Full-Finetuning of Generative Control Policies","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2603.12243","citing_title":"HandelBot: Real-World Piano Playing via Fast Adaptation of Dexterous Robot Policies","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18675","citing_title":"COOPO: Cyclic Offline-Online Policy Optimization Algorithm","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2507.07969","citing_title":"Reinforcement Learning with Action Chunking","ref_index":90,"is_internal_anchor":false},{"citing_arxiv_id":"2603.12243","citing_title":"HandelBot: Real-World Piano Playing via Fast Adaptation of Dexterous Robot Policies","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2603.15759","citing_title":"Simulation Distillation: Pretraining World Models in Simulation for Rapid Real-World Adaptation","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2603.15956","citing_title":"ExpertGen: Scalable Sim-to-Real Expert Policy Learning from Imperfect Behavior Priors","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08958","citing_title":"WOMBET: World Model-Based Experience Transfer for Robust and Sample-efficient Reinforcement Learning","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13966","citing_title":"Provably Efficient Offline-to-Online Value Adaptation with General Function Approximation","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03065","citing_title":"OGPO: Sample Efficient Full-Finetuning of Generative Control Policies","ref_index":191,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU","json":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU.json","graph_json":"https://pith.science/api/pith-number/XFRLHEB5KVWRYUIPE47OJV4RZU/graph.json","events_json":"https://pith.science/api/pith-number/XFRLHEB5KVWRYUIPE47OJV4RZU/events.json","paper":"https://pith.science/paper/XFRLHEB5"},"agent_actions":{"view_html":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU","download_json":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU.json","view_paper":"https://pith.science/paper/XFRLHEB5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.07762&json=true","fetch_graph":"https://pith.science/api/pith-number/XFRLHEB5KVWRYUIPE47OJV4RZU/graph.json","fetch_events":"https://pith.science/api/pith-number/XFRLHEB5KVWRYUIPE47OJV4RZU/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU/action/storage_attestation","attest_author":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU/action/author_attestation","sign_citation":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU/action/citation_signature","submit_replication":"https://pith.science/pith/XFRLHEB5KVWRYUIPE47OJV4RZU/action/replication_record"}},"created_at":"2026-07-05T11:30:39.857969+00:00","updated_at":"2026-07-05T11:30:39.857969+00:00"}