{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:MFIB4ZGCFJBF3UEPF3T2FITNBN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"36476bf4f1c0c61dc9393470f17a1c08176e6eed4c404f139a466f0b6b250ab5","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-04-21T08:09:13Z","title_canon_sha256":"a80d72f306e3f138b04b3428df5bee8762a2693c938769e3284fc45c99b69946"},"schema_version":"1.0","source":{"id":"2504.14945","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.14945","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"arxiv_version","alias_value":"2504.14945v5","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.14945","created_at":"2026-05-17T23:38:49Z"},{"alias_kind":"pith_short_12","alias_value":"MFIB4ZGCFJBF","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"MFIB4ZGCFJBF3UEP","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"MFIB4ZGC","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:ecef2603a7966951f496731faa27e79c25333fac2cca9a37f179d277a0c904aa","target":"graph","created_at":"2026-05-17T23:38:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Compared with previous RLVR methods, LUFFY achieves an over +6.4 average gain across six math benchmarks and an advantage of over +6.2 points in out-of-distribution tasks. Most significantly, we show that LUFFY successfully trains weak models in scenarios where on-policy RLVR completely fails."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That off-policy reasoning traces can be mixed with on-policy rollouts via regularized importance sampling without introducing harmful distribution shift or superficial imitation that would degrade the learned policy."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"LUFFY mixes off-policy reasoning traces into RLVR training via Mixed-Policy GRPO and regularized importance sampling, delivering over 6-point gains on math benchmarks and enabling training of weak models where on-policy RLVR fails."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"LUFFY mixes off-policy reasoning traces with on-policy rollouts to overcome the limits of standard RLVR in training reasoning models."}],"snapshot_sha256":"be06d94251a82b5fe6f308751045d5477423fe95369f697ebf5102bab74a1d6f"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"ad06fc1706691937f033154995f9c2b191515b50e75296522bf52595e1055527"},"paper":{"abstract_excerpt":"Recent advances in large reasoning models (LRMs) demonstrate that sophisticated behaviors such as multi-step reasoning and self-reflection can emerge via reinforcement learning with verifiable rewards~(\\textit{RLVR}). However, existing \\textit{RLVR} approaches are inherently ``on-policy'', limiting learning to a model's own outputs and failing to acquire reasoning abilities beyond its initial capabilities. To address this issue, we introduce \\textbf{LUFFY} (\\textbf{L}earning to reason \\textbf{U}nder o\\textbf{FF}-polic\\textbf{Y} guidance), a framework that augments \\textit{RLVR} with off-policy","authors_text":"Ganqu Cui, Jianhao Yan, Xiaoye Qu, Yafu Li, Yu Cheng, Yue Zhang, Zhi Wang, Zican Hu","cross_cats":["cs.AI","cs.CL"],"headline":"LUFFY mixes off-policy reasoning traces with on-policy rollouts to overcome the limits of standard RLVR in training reasoning models.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-04-21T08:09:13Z","title":"Learning to Reason under Off-Policy Guidance"},"references":{"count":65,"internal_anchors":18,"resolved_work":65,"sample":[{"cited_arxiv_id":"2412.16720","doi":"","is_internal_anchor":true,"ref_index":1,"title":"OpenAI o1 System Card","work_id":"68d3c334-0fc9-49e3-b7b0-a69afae933e2","year":2024},{"cited_arxiv_id":"2501.12948","doi":"","is_internal_anchor":true,"ref_index":2,"title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","work_id":"e6b75ad5-2877-4168-97c8-710407094d20","year":2025},{"cited_arxiv_id":"2501.12599","doi":"","is_internal_anchor":true,"ref_index":3,"title":"Kimi k1.5: Scaling Reinforcement Learning with LLMs","work_id":"bff96ab1-bd6a-4585-be23-74fdb51969c7","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Chi, Quoc V Le, and Denny Zhou","work_id":"15a94dfd-ad8c-4ad9-9b36-e591ddf4950d","year":2022},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"7b model and 8k examples: Emerging reasoning with reinforcement learning is both effective and efficient","work_id":"18772931-c0f8-4b45-bbb2-eb3f626eb2ee","year":2025}],"snapshot_sha256":"401a7fe1d67250ae59b59008c1a85145e85e42eb5baff34273cbe813d200b517"},"source":{"id":"2504.14945","kind":"arxiv","version":5},"verdict":{"created_at":"2026-05-15T23:13:37.696664Z","id":"9925b16a-9ca3-47ed-b0c7-8abeac542e55","model_set":{"reader":"grok-4.3"},"one_line_summary":"LUFFY mixes off-policy reasoning traces into RLVR training via Mixed-Policy GRPO and regularized importance sampling, delivering over 6-point gains on math benchmarks and enabling training of weak models where on-policy RLVR fails.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"LUFFY mixes off-policy reasoning traces with on-policy rollouts to overcome the limits of standard RLVR in training reasoning models.","strongest_claim":"Compared with previous RLVR methods, LUFFY achieves an over +6.4 average gain across six math benchmarks and an advantage of over +6.2 points in out-of-distribution tasks. Most significantly, we show that LUFFY successfully trains weak models in scenarios where on-policy RLVR completely fails.","weakest_assumption":"That off-policy reasoning traces can be mixed with on-policy rollouts via regularized importance sampling without introducing harmful distribution shift or superficial imitation that would degrade the learned policy."}},"verdict_id":"9925b16a-9ca3-47ed-b0c7-8abeac542e55"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bb0a4b7d8581526c46b1cbeabdd0a85456a4210f3096aca17ad114264b62943f","target":"record","created_at":"2026-05-17T23:38:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"36476bf4f1c0c61dc9393470f17a1c08176e6eed4c404f139a466f0b6b250ab5","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-04-21T08:09:13Z","title_canon_sha256":"a80d72f306e3f138b04b3428df5bee8762a2693c938769e3284fc45c99b69946"},"schema_version":"1.0","source":{"id":"2504.14945","kind":"arxiv","version":5}},"canonical_sha256":"61501e64c22a425dd08f2ee7a2a26d0b79e76dd1293c56cebf08c0349ba48cd4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"61501e64c22a425dd08f2ee7a2a26d0b79e76dd1293c56cebf08c0349ba48cd4","first_computed_at":"2026-05-17T23:38:49.812561Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:49.812561Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+XSUBs2nnRrFQaZXD454JsQghr4l2gZByQVSr10f5HIbNpolE5+nFE9ET8rf5FCti/W4YO0/lP/bD6izoPPoCg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:49.813136Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.14945","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bb0a4b7d8581526c46b1cbeabdd0a85456a4210f3096aca17ad114264b62943f","sha256:ecef2603a7966951f496731faa27e79c25333fac2cca9a37f179d277a0c904aa"],"state_sha256":"15f1389503cc996b9beb23583638c958cd7e1775cc13970b327f172ec52b003c"}