{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:WOPET2HVNMRIUBJIDORQU55FCD","short_pith_number":"pith:WOPET2HV","schema_version":"1.0","canonical_sha256":"b39e49e8f56b228a05281ba30a77a510ecd0413c61d68b82755914bed7471fa9","source":{"kind":"arxiv","id":"2311.05821","version":1},"attestation_state":"computed","paper":{"title":"Let's Reinforce Step by Step","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anna Rumshisky, Sarah Pan, Sherin Muckatira, Vladislav Lialin","submitted_at":"2023-11-10T01:35:51Z","abstract_excerpt":"While recent advances have boosted LM proficiency in linguistic benchmarks, LMs consistently struggle to reason correctly on complex tasks like mathematics. We turn to Reinforcement Learning from Human Feedback (RLHF) as a method with which to shape model reasoning processes. In particular, we explore two reward schemes, outcome-supervised reward models (ORMs) and process-supervised reward models (PRMs), to optimize for logical reasoning. Our results show that the fine-grained reward provided by PRM-based methods enhances accuracy on simple mathematical reasoning (GSM8K) while, unexpectedly, r"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2311.05821","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-10T01:35:51Z","cross_cats_sorted":[],"title_canon_sha256":"a3fc4a39c27169790563790fb307a5a4ed640730195579d2389031c695cf9e29","abstract_canon_sha256":"92a17dcb8d1b20647d440d5dee92612592558eb26f0b7f1bde567768c18bca1c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:11:23.338750Z","signature_b64":"5Y6xkyQDYngDt79Ll3kKHd0m1rdHwh7G2tK+rwp7XPDT+fWlIHXvsyYkxQBDn8HrsXUjFjmVJkM9G1jolexYBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b39e49e8f56b228a05281ba30a77a510ecd0413c61d68b82755914bed7471fa9","last_reissued_at":"2026-07-05T07:11:23.338312Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:11:23.338312Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Let's Reinforce Step by Step","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anna Rumshisky, Sarah Pan, Sherin Muckatira, Vladislav Lialin","submitted_at":"2023-11-10T01:35:51Z","abstract_excerpt":"While recent advances have boosted LM proficiency in linguistic benchmarks, LMs consistently struggle to reason correctly on complex tasks like mathematics. We turn to Reinforcement Learning from Human Feedback (RLHF) as a method with which to shape model reasoning processes. In particular, we explore two reward schemes, outcome-supervised reward models (ORMs) and process-supervised reward models (PRMs), to optimize for logical reasoning. Our results show that the fine-grained reward provided by PRM-based methods enhances accuracy on simple mathematical reasoning (GSM8K) while, unexpectedly, r"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.05821","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.05821/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2311.05821","created_at":"2026-07-05T07:11:23.338379+00:00"},{"alias_kind":"arxiv_version","alias_value":"2311.05821v1","created_at":"2026-07-05T07:11:23.338379+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.05821","created_at":"2026-07-05T07:11:23.338379+00:00"},{"alias_kind":"pith_short_12","alias_value":"WOPET2HVNMRI","created_at":"2026-07-05T07:11:23.338379+00:00"},{"alias_kind":"pith_short_16","alias_value":"WOPET2HVNMRIUBJI","created_at":"2026-07-05T07:11:23.338379+00:00"},{"alias_kind":"pith_short_8","alias_value":"WOPET2HV","created_at":"2026-07-05T07:11:23.338379+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2408.07199","citing_title":"Agent Q: Advanced Reasoning and Learning for Autonomous AI Agents","ref_index":213,"is_internal_anchor":false},{"citing_arxiv_id":"2312.08935","citing_title":"Math-Shepherd: Verify and Reinforce LLMs Step-by-step without Human Annotations","ref_index":76,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD","json":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD.json","graph_json":"https://pith.science/api/pith-number/WOPET2HVNMRIUBJIDORQU55FCD/graph.json","events_json":"https://pith.science/api/pith-number/WOPET2HVNMRIUBJIDORQU55FCD/events.json","paper":"https://pith.science/paper/WOPET2HV"},"agent_actions":{"view_html":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD","download_json":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD.json","view_paper":"https://pith.science/paper/WOPET2HV","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2311.05821&json=true","fetch_graph":"https://pith.science/api/pith-number/WOPET2HVNMRIUBJIDORQU55FCD/graph.json","fetch_events":"https://pith.science/api/pith-number/WOPET2HVNMRIUBJIDORQU55FCD/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD/action/timestamp_anchor","attest_storage":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD/action/storage_attestation","attest_author":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD/action/author_attestation","sign_citation":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD/action/citation_signature","submit_replication":"https://pith.science/pith/WOPET2HVNMRIUBJIDORQU55FCD/action/replication_record"}},"created_at":"2026-07-05T07:11:23.338379+00:00","updated_at":"2026-07-05T07:11:23.338379+00:00"}