{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:3PD4QSV2P64X7KHEPXFBAXM24H","short_pith_number":"pith:3PD4QSV2","schema_version":"1.0","canonical_sha256":"dbc7c84aba7fb97fa8e47dca105d9ae1fa350de29fb5a13d5f78a5c0e50569b9","source":{"kind":"arxiv","id":"2505.17218","version":1},"attestation_state":"computed","paper":{"title":"Effective Reinforcement Learning for Reasoning in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Insup Lee, Lianghuan Huang, Osbert Bastani, Sagnik Anupam, Shuo Li","submitted_at":"2025-05-22T18:48:09Z","abstract_excerpt":"Reinforcement learning (RL) has emerged as a promising strategy for improving the reasoning capabilities of language models (LMs) in domains such as mathematics and coding. However, most modern RL algorithms were designed to target robotics applications, which differ significantly from LM reasoning. We analyze RL algorithm design decisions for LM reasoning, for both accuracy and computational efficiency, focusing on relatively small models due to computational constraints. Our findings are: (i) on-policy RL significantly outperforms supervised fine-tuning (SFT), (ii) PPO-based off-policy updat"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.17218","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-22T18:48:09Z","cross_cats_sorted":[],"title_canon_sha256":"80f7fa0a168650a3ef7628271cc1d5c33b9a4a5e8a42319be3d8829f3b16b424","abstract_canon_sha256":"043f806e687e8f205e7a5e664a22c90323e3489b29eba8d05f8ebbac77833cb9"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:07:49.039091Z","signature_b64":"/AHNSguZdRZsV2WZ2GkxMhKKxE0jZMApsbQtB07Ck2qi+asQ0OG4I4opfm7s4MQ6Oswk0pfNnFCwoaBFep3RCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dbc7c84aba7fb97fa8e47dca105d9ae1fa350de29fb5a13d5f78a5c0e50569b9","last_reissued_at":"2026-07-05T11:07:49.038531Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:07:49.038531Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Effective Reinforcement Learning for Reasoning in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Insup Lee, Lianghuan Huang, Osbert Bastani, Sagnik Anupam, Shuo Li","submitted_at":"2025-05-22T18:48:09Z","abstract_excerpt":"Reinforcement learning (RL) has emerged as a promising strategy for improving the reasoning capabilities of language models (LMs) in domains such as mathematics and coding. However, most modern RL algorithms were designed to target robotics applications, which differ significantly from LM reasoning. We analyze RL algorithm design decisions for LM reasoning, for both accuracy and computational efficiency, focusing on relatively small models due to computational constraints. Our findings are: (i) on-policy RL significantly outperforms supervised fine-tuning (SFT), (ii) PPO-based off-policy updat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.17218","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.17218/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.17218","created_at":"2026-07-05T11:07:49.038612+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.17218v1","created_at":"2026-07-05T11:07:49.038612+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.17218","created_at":"2026-07-05T11:07:49.038612+00:00"},{"alias_kind":"pith_short_12","alias_value":"3PD4QSV2P64X","created_at":"2026-07-05T11:07:49.038612+00:00"},{"alias_kind":"pith_short_16","alias_value":"3PD4QSV2P64X7KHE","created_at":"2026-07-05T11:07:49.038612+00:00"},{"alias_kind":"pith_short_8","alias_value":"3PD4QSV2","created_at":"2026-07-05T11:07:49.038612+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.29758","citing_title":"PS-PPO: Prefix-Sampling PPO for Critic-Free RLHF","ref_index":21,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H","json":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H.json","graph_json":"https://pith.science/api/pith-number/3PD4QSV2P64X7KHEPXFBAXM24H/graph.json","events_json":"https://pith.science/api/pith-number/3PD4QSV2P64X7KHEPXFBAXM24H/events.json","paper":"https://pith.science/paper/3PD4QSV2"},"agent_actions":{"view_html":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H","download_json":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H.json","view_paper":"https://pith.science/paper/3PD4QSV2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.17218&json=true","fetch_graph":"https://pith.science/api/pith-number/3PD4QSV2P64X7KHEPXFBAXM24H/graph.json","fetch_events":"https://pith.science/api/pith-number/3PD4QSV2P64X7KHEPXFBAXM24H/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H/action/storage_attestation","attest_author":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H/action/author_attestation","sign_citation":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H/action/citation_signature","submit_replication":"https://pith.science/pith/3PD4QSV2P64X7KHEPXFBAXM24H/action/replication_record"}},"created_at":"2026-07-05T11:07:49.038612+00:00","updated_at":"2026-07-05T11:07:49.038612+00:00"}