{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:HA273CXNLRK6Z3ZEUNKIWPUO2O","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"170c1475e304016e31622d2bc8f51339fe403395782174af126da8e0c1c24042","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-18T14:23:37Z","title_canon_sha256":"289d6d612117f3e318153231d480fc23b6df73de45ce703d7e21c3097f5704f6"},"schema_version":"1.0","source":{"id":"2503.14286","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.14286","created_at":"2026-07-05T10:34:43Z"},{"alias_kind":"arxiv_version","alias_value":"2503.14286v2","created_at":"2026-07-05T10:34:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.14286","created_at":"2026-07-05T10:34:43Z"},{"alias_kind":"pith_short_12","alias_value":"HA273CXNLRK6","created_at":"2026-07-05T10:34:43Z"},{"alias_kind":"pith_short_16","alias_value":"HA273CXNLRK6Z3ZE","created_at":"2026-07-05T10:34:43Z"},{"alias_kind":"pith_short_8","alias_value":"HA273CXN","created_at":"2026-07-05T10:34:43Z"}],"graph_snapshots":[{"event_id":"sha256:b4f51914fa34169d2193810d9678c890f1ffcae56c7ad741a95370beab1b5df6","target":"graph","created_at":"2026-07-05T10:34:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.14286/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose a new algorithm for fine-tuning large language models using reinforcement learning. Tapered Off-Policy REINFORCE (TOPR) uses an asymmetric, tapered variant of importance sampling to speed up learning while maintaining stable learning dynamics, even without the use of KL regularization. TOPR can be applied in a fully offline fashion, allows the handling of positive and negative examples in a unified framework, and benefits from the implementational simplicity that is typical of Monte Carlo algorithms. We demonstrate the effectiveness of our approach with a series of experiments on th","authors_text":"Alex Fr\\'echette, Arnaud Bergeron, Carolyne Pelletier, Eric Thibodeau-Laufer, Jonathan Lebensold, Joshua Greaves, Marc G. Bellemare, Nicolas Le Roux, Sam Work, S\\'andor Toth","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-18T14:23:37Z","title":"Tapered Off-Policy REINFORCE: Stable and efficient reinforcement learning for LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.14286","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3d62f89714acc23fc28a8451ec1933da3036de2f6bbb65a20a8cf64aa2825fb9","target":"record","created_at":"2026-07-05T10:34:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"170c1475e304016e31622d2bc8f51339fe403395782174af126da8e0c1c24042","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-18T14:23:37Z","title_canon_sha256":"289d6d612117f3e318153231d480fc23b6df73de45ce703d7e21c3097f5704f6"},"schema_version":"1.0","source":{"id":"2503.14286","kind":"arxiv","version":2}},"canonical_sha256":"3835fd8aed5c55ecef24a3548b3e8ed38a7c47f9e50f1ce6f266ec683b7c631b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3835fd8aed5c55ecef24a3548b3e8ed38a7c47f9e50f1ce6f266ec683b7c631b","first_computed_at":"2026-07-05T10:34:43.804692Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:34:43.804692Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ceW9KEUz1nzQp/Gf7mYnGrQQtlqjDEKkSoiKgPHaSCYv7Lg5TASFQs8rZxTSOrxvxkFkew+8lit5ewKmifQMAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:34:43.805374Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.14286","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3d62f89714acc23fc28a8451ec1933da3036de2f6bbb65a20a8cf64aa2825fb9","sha256:b4f51914fa34169d2193810d9678c890f1ffcae56c7ad741a95370beab1b5df6"],"state_sha256":"33311091f9142a6d00a12c49f17cd25acd4476d55ece7b0ae2edb391355407be"}