{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:VL5O74ICDVJNMAXR7BSO4BH5K2","short_pith_number":"pith:VL5O74IC","schema_version":"1.0","canonical_sha256":"aafaeff1021d52d602f1f864ee04fd5692fb36a6f0d149dc3c00e17c5ab80b46","source":{"kind":"arxiv","id":"2405.00451","version":2},"attestation_state":"computed","paper":{"title":"Monte Carlo Tree Search Boosts Reasoning via Iterative Preference Learning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Anirudh Goyal, Kenji Kawaguchi, Michael Shieh, Min-Yen Kan, Timothy P. Lillicrap, Wenyue Zheng, Yuxi Xie","submitted_at":"2024-05-01T11:10:24Z","abstract_excerpt":"We introduce an approach aimed at enhancing the reasoning capabilities of Large Language Models (LLMs) through an iterative preference learning process inspired by the successful strategy employed by AlphaZero. Our work leverages Monte Carlo Tree Search (MCTS) to iteratively collect preference data, utilizing its look-ahead ability to break down instance-level rewards into more granular step-level signals. To enhance consistency in intermediate steps, we combine outcome validation and stepwise self-evaluation, continually updating the quality assessment of newly generated data. The proposed al"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.00451","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-05-01T11:10:24Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"12c09afcab6ead52f8af2a4a107bc5592ef43c6cc98cffe3e10e8ce1cd8f83f3","abstract_canon_sha256":"0a0acfb0680fc60973fb1913fd32dec6b8306ec63f770cf38d56b8641aeb451e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:33:01.840679Z","signature_b64":"Rcb9dszi9xbmSkO+T5rw5uFhsLrQzHxBmq1Prr9e9pQhKlFPopr0kK7MlS7R6mRxNV2hkuq6mUIi+LGfBYCqCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"aafaeff1021d52d602f1f864ee04fd5692fb36a6f0d149dc3c00e17c5ab80b46","last_reissued_at":"2026-07-05T08:33:01.840151Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:33:01.840151Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Monte Carlo Tree Search Boosts Reasoning via Iterative Preference Learning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Anirudh Goyal, Kenji Kawaguchi, Michael Shieh, Min-Yen Kan, Timothy P. Lillicrap, Wenyue Zheng, Yuxi Xie","submitted_at":"2024-05-01T11:10:24Z","abstract_excerpt":"We introduce an approach aimed at enhancing the reasoning capabilities of Large Language Models (LLMs) through an iterative preference learning process inspired by the successful strategy employed by AlphaZero. Our work leverages Monte Carlo Tree Search (MCTS) to iteratively collect preference data, utilizing its look-ahead ability to break down instance-level rewards into more granular step-level signals. To enhance consistency in intermediate steps, we combine outcome validation and stepwise self-evaluation, continually updating the quality assessment of newly generated data. The proposed al"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00451","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.00451/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.00451","created_at":"2026-07-05T08:33:01.840210+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.00451v2","created_at":"2026-07-05T08:33:01.840210+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00451","created_at":"2026-07-05T08:33:01.840210+00:00"},{"alias_kind":"pith_short_12","alias_value":"VL5O74ICDVJN","created_at":"2026-07-05T08:33:01.840210+00:00"},{"alias_kind":"pith_short_16","alias_value":"VL5O74ICDVJNMAXR","created_at":"2026-07-05T08:33:01.840210+00:00"},{"alias_kind":"pith_short_8","alias_value":"VL5O74IC","created_at":"2026-07-05T08:33:01.840210+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":35,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25354","citing_title":"Efficient and Trainable Language Model Test-Time Scaling via Local Branch Routing","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21943","citing_title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","ref_index":236,"is_internal_anchor":false},{"citing_arxiv_id":"2606.13316","citing_title":"ReSum: Synergizing LLM Reasoning and Summarization with Reinforcement Learning","ref_index":63,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12384","citing_title":"APPO: Agentic Procedural Policy Optimization","ref_index":86,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11119","citing_title":"TRACE: A Unified Rollout Budget Allocation Framework for Efficient Agentic Reinforcement Learning","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08231","citing_title":"Test-Time Scaling in Multimodal Foundation Models: A Comprehensive Survey of Generation and Reasoning","ref_index":97,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05464","citing_title":"Step-by-Step Optimization-like Reasoning in LLMs over Expanding Search Spaces","ref_index":37,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01667","citing_title":"ATLAS: Agentic Test-time Learning-to-Allocate Scaling","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31748","citing_title":"Addressing Over-Refusal in LLMs with Competing Rewards","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2606.25354","citing_title":"Efficient and Trainable Language Model Test-Time Scaling via Local Branch Routing","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10913","citing_title":"Shepherd: Enabling Programmable Meta-Agents via Reversible Agentic Execution Traces","ref_index":48,"is_internal_anchor":false},{"citing_arxiv_id":"2606.20599","citing_title":"Beyond Fixed Budgets: Characterizing the Inelasticity and Limitations of Tree-of-Thought Reasoning Strategies","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24326","citing_title":"ScaleAcross Explorer: Exploring Communication Optimization for Scale-Across AI Model Training","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2502.00955","citing_title":"Efficient Multi-Agent System Training with Data Influence-Oriented Tree Search","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2504.02181","citing_title":"A Survey of Scaling in Large Language Model Reasoning","ref_index":231,"is_internal_anchor":false},{"citing_arxiv_id":"2504.13818","citing_title":"Not All Rollouts are Useful: Down-Sampling Rollouts in LLM Reinforcement Learning","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2501.09732","citing_title":"Inference-Time Scaling for Diffusion Models beyond Scaling Denoising Steps","ref_index":91,"is_internal_anchor":false},{"citing_arxiv_id":"2507.04736","citing_title":"ChipSeek: Optimizing Verilog Generation via EDA-Integrated Reinforcement Learning","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2509.21743","citing_title":"Retrieval-of-Thought: Efficient Reasoning via Reusing Thoughts","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2510.08592","citing_title":"Less Diverse, Less Safe: The Indirect But Pervasive Risk of Test-Time Scaling in Large Language Models","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2512.07461","citing_title":"Native Parallel Reasoner: Reasoning in Parallelism via Self-Distilled Reinforcement Learning","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2510.13786","citing_title":"The Art of Scaling Reinforcement Learning Compute for LLMs","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2503.07536","citing_title":"LMM-R1: Empowering 3B LMMs with Strong Reasoning Abilities Through Two-Stage Rule-Based RL","ref_index":84,"is_internal_anchor":false},{"citing_arxiv_id":"2501.09686","citing_title":"Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models","ref_index":167,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10195","citing_title":"Breaking the Reward Barrier: Accelerating Tree-of-Thought Reasoning via Speculative Exploration","ref_index":61,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2","json":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2.json","graph_json":"https://pith.science/api/pith-number/VL5O74ICDVJNMAXR7BSO4BH5K2/graph.json","events_json":"https://pith.science/api/pith-number/VL5O74ICDVJNMAXR7BSO4BH5K2/events.json","paper":"https://pith.science/paper/VL5O74IC"},"agent_actions":{"view_html":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2","download_json":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2.json","view_paper":"https://pith.science/paper/VL5O74IC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.00451&json=true","fetch_graph":"https://pith.science/api/pith-number/VL5O74ICDVJNMAXR7BSO4BH5K2/graph.json","fetch_events":"https://pith.science/api/pith-number/VL5O74ICDVJNMAXR7BSO4BH5K2/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2/action/timestamp_anchor","attest_storage":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2/action/storage_attestation","attest_author":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2/action/author_attestation","sign_citation":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2/action/citation_signature","submit_replication":"https://pith.science/pith/VL5O74ICDVJNMAXR7BSO4BH5K2/action/replication_record"}},"created_at":"2026-07-05T08:33:01.840210+00:00","updated_at":"2026-07-05T08:33:01.840210+00:00"}