{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:HA273CXNLRK6Z3ZEUNKIWPUO2O","short_pith_number":"pith:HA273CXN","schema_version":"1.0","canonical_sha256":"3835fd8aed5c55ecef24a3548b3e8ed38a7c47f9e50f1ce6f266ec683b7c631b","source":{"kind":"arxiv","id":"2503.14286","version":2},"attestation_state":"computed","paper":{"title":"Tapered Off-Policy REINFORCE: Stable and efficient reinforcement learning for LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alex Fr\\'echette, Arnaud Bergeron, Carolyne Pelletier, Eric Thibodeau-Laufer, Jonathan Lebensold, Joshua Greaves, Marc G. Bellemare, Nicolas Le Roux, Sam Work, S\\'andor Toth","submitted_at":"2025-03-18T14:23:37Z","abstract_excerpt":"We propose a new algorithm for fine-tuning large language models using reinforcement learning. Tapered Off-Policy REINFORCE (TOPR) uses an asymmetric, tapered variant of importance sampling to speed up learning while maintaining stable learning dynamics, even without the use of KL regularization. TOPR can be applied in a fully offline fashion, allows the handling of positive and negative examples in a unified framework, and benefits from the implementational simplicity that is typical of Monte Carlo algorithms. We demonstrate the effectiveness of our approach with a series of experiments on th"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2503.14286","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-18T14:23:37Z","cross_cats_sorted":[],"title_canon_sha256":"289d6d612117f3e318153231d480fc23b6df73de45ce703d7e21c3097f5704f6","abstract_canon_sha256":"170c1475e304016e31622d2bc8f51339fe403395782174af126da8e0c1c24042"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:34:43.805374Z","signature_b64":"ceW9KEUz1nzQp/Gf7mYnGrQQtlqjDEKkSoiKgPHaSCYv7Lg5TASFQs8rZxTSOrxvxkFkew+8lit5ewKmifQMAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3835fd8aed5c55ecef24a3548b3e8ed38a7c47f9e50f1ce6f266ec683b7c631b","last_reissued_at":"2026-07-05T10:34:43.804692Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:34:43.804692Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Tapered Off-Policy REINFORCE: Stable and efficient reinforcement learning for LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alex Fr\\'echette, Arnaud Bergeron, Carolyne Pelletier, Eric Thibodeau-Laufer, Jonathan Lebensold, Joshua Greaves, Marc G. Bellemare, Nicolas Le Roux, Sam Work, S\\'andor Toth","submitted_at":"2025-03-18T14:23:37Z","abstract_excerpt":"We propose a new algorithm for fine-tuning large language models using reinforcement learning. Tapered Off-Policy REINFORCE (TOPR) uses an asymmetric, tapered variant of importance sampling to speed up learning while maintaining stable learning dynamics, even without the use of KL regularization. TOPR can be applied in a fully offline fashion, allows the handling of positive and negative examples in a unified framework, and benefits from the implementational simplicity that is typical of Monte Carlo algorithms. We demonstrate the effectiveness of our approach with a series of experiments on th"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.14286","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.14286/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2503.14286","created_at":"2026-07-05T10:34:43.804769+00:00"},{"alias_kind":"arxiv_version","alias_value":"2503.14286v2","created_at":"2026-07-05T10:34:43.804769+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.14286","created_at":"2026-07-05T10:34:43.804769+00:00"},{"alias_kind":"pith_short_12","alias_value":"HA273CXNLRK6","created_at":"2026-07-05T10:34:43.804769+00:00"},{"alias_kind":"pith_short_16","alias_value":"HA273CXNLRK6Z3ZE","created_at":"2026-07-05T10:34:43.804769+00:00"},{"alias_kind":"pith_short_8","alias_value":"HA273CXN","created_at":"2026-07-05T10:34:43.804769+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":14,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.06987","citing_title":"UP: Unbounded Positive Asymmetric Optimization for Breaking the Exploration-Stability Dilemma","ref_index":25,"is_internal_anchor":true},{"citing_arxiv_id":"2606.22570","citing_title":"What are Key Factors for Updates in RL for LLM Reasoning?","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21943","citing_title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","ref_index":161,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04560","citing_title":"Rollout-Level Advantage-Prioritized Experience Replay for GRPO","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01249","citing_title":"Trust Region On-Policy Distillation","ref_index":218,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25582","citing_title":"Extreme Region Policy Distillation","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.26784","citing_title":"Ratio-Variance Regularized Policy Optimization","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":136,"is_internal_anchor":false},{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":125,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17570","citing_title":"How Off-Policy Can GRPO Be? Mu-GRPO for Efficient LLM Reinforcement Learning","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19282","citing_title":"Rethinking Muon Beyond Pretraining: Spectral Failures and High-Pass Remedies for VLA and RLVR","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2509.25424","citing_title":"Polychromic Objectives for Reinforcement Learning","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12070","citing_title":"Missing Old Logits in Asynchronous Agentic RL: Semantic Mismatch and Repair Methods for Off-Policy Correction","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16918","citing_title":"Freshness-Aware Prioritized Experience Replay for LLM/VLM Reinforcement Learning","ref_index":15,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O","json":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O.json","graph_json":"https://pith.science/api/pith-number/HA273CXNLRK6Z3ZEUNKIWPUO2O/graph.json","events_json":"https://pith.science/api/pith-number/HA273CXNLRK6Z3ZEUNKIWPUO2O/events.json","paper":"https://pith.science/paper/HA273CXN"},"agent_actions":{"view_html":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O","download_json":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O.json","view_paper":"https://pith.science/paper/HA273CXN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2503.14286&json=true","fetch_graph":"https://pith.science/api/pith-number/HA273CXNLRK6Z3ZEUNKIWPUO2O/graph.json","fetch_events":"https://pith.science/api/pith-number/HA273CXNLRK6Z3ZEUNKIWPUO2O/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O/action/timestamp_anchor","attest_storage":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O/action/storage_attestation","attest_author":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O/action/author_attestation","sign_citation":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O/action/citation_signature","submit_replication":"https://pith.science/pith/HA273CXNLRK6Z3ZEUNKIWPUO2O/action/replication_record"}},"created_at":"2026-07-05T10:34:43.804769+00:00","updated_at":"2026-07-05T10:34:43.804769+00:00"}