{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:ZEVSCJFXC3UD4O6RPU66X7GL4Q","short_pith_number":"pith:ZEVSCJFX","schema_version":"1.0","canonical_sha256":"c92b2124b716e83e3bd17d3debfccbe41f91b99321963f6fc0b76f59b6a38064","source":{"kind":"arxiv","id":"2410.01679","version":2},"attestation_state":"computed","paper":{"title":"VinePPO: Refining Credit Assignment in RL Training of LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Aaron Courville, Alessandro Sordoni, Amirhossein Kazemnejad, Eva Portelance, Milad Aghajohari, Nicolas Le Roux, Siva Reddy","submitted_at":"2024-10-02T15:49:30Z","abstract_excerpt":"Large language models (LLMs) are increasingly applied to complex reasoning tasks that require executing several complex steps before receiving any reward. Properly assigning credit to these steps is essential for enhancing model performance. Proximal Policy Optimization (PPO), a common reinforcement learning (RL) algorithm used for LLM finetuning, employs value networks to tackle credit assignment. However, recent approaches achieve strong results without it, raising questions about the efficacy of value networks in practice. In this work, we systematically evaluate the efficacy of value netwo"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.01679","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T15:49:30Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"0de23e876e2e3b01639995888a94a28e5037a8e7e4dd84851122fe6e1d59f340","abstract_canon_sha256":"3cce013654c15c55fc0af6e9a8751056231adf9e953e894eba76530a6a5b6a42"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:15:17.722397Z","signature_b64":"GSmbkacmQFHD4be8acrPnkfBkGZWB/NTVqja96ABU/z+XCcfd3+jD3Kgz/BYOK3Y2RG3vbDngFKlrxMAOLPTDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c92b2124b716e83e3bd17d3debfccbe41f91b99321963f6fc0b76f59b6a38064","last_reissued_at":"2026-07-05T11:15:17.721862Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:15:17.721862Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"VinePPO: Refining Credit Assignment in RL Training of LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Aaron Courville, Alessandro Sordoni, Amirhossein Kazemnejad, Eva Portelance, Milad Aghajohari, Nicolas Le Roux, Siva Reddy","submitted_at":"2024-10-02T15:49:30Z","abstract_excerpt":"Large language models (LLMs) are increasingly applied to complex reasoning tasks that require executing several complex steps before receiving any reward. Properly assigning credit to these steps is essential for enhancing model performance. Proximal Policy Optimization (PPO), a common reinforcement learning (RL) algorithm used for LLM finetuning, employs value networks to tackle credit assignment. However, recent approaches achieve strong results without it, raising questions about the efficacy of value networks in practice. In this work, we systematically evaluate the efficacy of value netwo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.01679","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.01679/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.01679","created_at":"2026-07-05T11:15:17.721928+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.01679v2","created_at":"2026-07-05T11:15:17.721928+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.01679","created_at":"2026-07-05T11:15:17.721928+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZEVSCJFXC3UD","created_at":"2026-07-05T11:15:17.721928+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZEVSCJFXC3UD4O6R","created_at":"2026-07-05T11:15:17.721928+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZEVSCJFX","created_at":"2026-07-05T11:15:17.721928+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":33,"internal_anchor_count":2,"sample":[{"citing_arxiv_id":"2607.07435","citing_title":"RLVP: Penalize the Path, Reward the Outcome","ref_index":9,"is_internal_anchor":true},{"citing_arxiv_id":"2607.05394","citing_title":"Weak-to-Strong Generalization via Direct On-Policy Distillation","ref_index":105,"is_internal_anchor":true},{"citing_arxiv_id":"2606.11209","citing_title":"ProcessThinker: Enhancing Multi-modal Large Language Models Reasoning via Rollout-based Process Reward","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21943","citing_title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","ref_index":87,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04036","citing_title":"Self-Distilled Policy Gradient","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01281","citing_title":"RLVR without Ineffective Samples: Group Prioritized Off-Policy Optimization for LLM Reasoning","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00257","citing_title":"ARCA: Adapter-Residual Credit Assignment When Token Signals Degenerate","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29745","citing_title":"ECHO: Learning Epistemically Adaptive Language Agents with Turn-Level Credit","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29476","citing_title":"CRAFT: Counterfactual Credit Assignment from Free Sibling Rollouts for Self-Distilled Agentic Reinforcement Learning","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27788","citing_title":"Knowing When to Ask: Segment-Level Credit Assignment for LLM Tool Use","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2502.12272","citing_title":"Learning to Reason at the Frontier of Learnability","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2503.14476","citing_title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":165,"is_internal_anchor":false},{"citing_arxiv_id":"2504.13818","citing_title":"Not All Rollouts are Useful: Down-Sampling Rollouts in LLM Reinforcement Learning","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21467","citing_title":"DelTA: Discriminative Token Credit Assignment for Reinforcement Learning from Verifiable Rewards","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21235","citing_title":"LamPO: A Lambda Style Policy Optimization for Reasoning Language Models","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2410.08146","citing_title":"Rewarding Progress: Scaling Automated Process Verifiers for LLM Reasoning","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15113","citing_title":"Learning from Language Feedback via Variational Policy Distillation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27859","citing_title":"Rethinking Agentic Reinforcement Learning In Large Language Models","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2509.02547","citing_title":"The Landscape of Agentic Reinforcement Learning for LLMs: A Survey","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2509.25424","citing_title":"Polychromic Objectives for Reinforcement Learning","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2602.03452","citing_title":"Beyond Variance: Prompt-Efficient RLVR via Rare-Event Amplification and Bidirectional Pairing","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2501.09686","citing_title":"Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models","ref_index":61,"is_internal_anchor":false},{"citing_arxiv_id":"2504.20571","citing_title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14558","citing_title":"Resolving Action Bottleneck: Agentic Reinforcement Learning Informed by Token-Level Energy","ref_index":11,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q","json":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q.json","graph_json":"https://pith.science/api/pith-number/ZEVSCJFXC3UD4O6RPU66X7GL4Q/graph.json","events_json":"https://pith.science/api/pith-number/ZEVSCJFXC3UD4O6RPU66X7GL4Q/events.json","paper":"https://pith.science/paper/ZEVSCJFX"},"agent_actions":{"view_html":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q","download_json":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q.json","view_paper":"https://pith.science/paper/ZEVSCJFX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.01679&json=true","fetch_graph":"https://pith.science/api/pith-number/ZEVSCJFXC3UD4O6RPU66X7GL4Q/graph.json","fetch_events":"https://pith.science/api/pith-number/ZEVSCJFXC3UD4O6RPU66X7GL4Q/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q/action/storage_attestation","attest_author":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q/action/author_attestation","sign_citation":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q/action/citation_signature","submit_replication":"https://pith.science/pith/ZEVSCJFXC3UD4O6RPU66X7GL4Q/action/replication_record"}},"created_at":"2026-07-05T11:15:17.721928+00:00","updated_at":"2026-07-05T11:15:17.721928+00:00"}