{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:PCAKBK2UR5BY3YVYOQTTYMPISP","short_pith_number":"pith:PCAKBK2U","schema_version":"1.0","canonical_sha256":"7880a0ab548f438de2b874273c31e893c824ef012f4a88b4422e166b0f7f1740","source":{"kind":"arxiv","id":"2412.18279","version":1},"attestation_state":"computed","paper":{"title":"Improving Multi-Step Reasoning Abilities of Large Language Models with Direct Advantage Policy Optimization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Chaojie Wang, Chris Yuhao Liu, Jiacai Liu, Liang Zeng, Rui Yan, Yahui Zhou, Yang Liu, Yiwen Sun","submitted_at":"2024-12-24T08:39:35Z","abstract_excerpt":"The role of reinforcement learning (RL) in enhancing the reasoning of large language models (LLMs) is becoming increasingly significant. Despite the success of RL in many scenarios, there are still many challenges in improving the reasoning of LLMs. One challenge is the sparse reward, which makes optimization difficult for RL and necessitates a large amount of data samples. Another challenge stems from the inherent instability of RL, particularly when using Actor-Critic (AC) methods to derive optimal policies, which often leads to unstable training processes. To address these issues, we introd"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.18279","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-12-24T08:39:35Z","cross_cats_sorted":[],"title_canon_sha256":"5295e12e227953e659bbeedc40fb6f2a3fc7e8b6241696d4edc74b67884884c0","abstract_canon_sha256":"7898e041e819958ef9fd06e7b144278e7032ec3299d2dc1620b39fbeb8c99649"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:53:53.214538Z","signature_b64":"qMC/j+zPyYmHYT85fMDta+0iCGfLrQ5fLnUUAajFdqgVaXdbLUXJGrtjDxzKAlFJWKIORoScMnVToLVczpnNAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7880a0ab548f438de2b874273c31e893c824ef012f4a88b4422e166b0f7f1740","last_reissued_at":"2026-07-05T09:53:53.214048Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:53:53.214048Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Improving Multi-Step Reasoning Abilities of Large Language Models with Direct Advantage Policy Optimization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Chaojie Wang, Chris Yuhao Liu, Jiacai Liu, Liang Zeng, Rui Yan, Yahui Zhou, Yang Liu, Yiwen Sun","submitted_at":"2024-12-24T08:39:35Z","abstract_excerpt":"The role of reinforcement learning (RL) in enhancing the reasoning of large language models (LLMs) is becoming increasingly significant. Despite the success of RL in many scenarios, there are still many challenges in improving the reasoning of LLMs. One challenge is the sparse reward, which makes optimization difficult for RL and necessitates a large amount of data samples. Another challenge stems from the inherent instability of RL, particularly when using Actor-Critic (AC) methods to derive optimal policies, which often leads to unstable training processes. To address these issues, we introd"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.18279","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.18279/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.18279","created_at":"2026-07-05T09:53:53.214105+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.18279v1","created_at":"2026-07-05T09:53:53.214105+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.18279","created_at":"2026-07-05T09:53:53.214105+00:00"},{"alias_kind":"pith_short_12","alias_value":"PCAKBK2UR5BY","created_at":"2026-07-05T09:53:53.214105+00:00"},{"alias_kind":"pith_short_16","alias_value":"PCAKBK2UR5BY3YVY","created_at":"2026-07-05T09:53:53.214105+00:00"},{"alias_kind":"pith_short_8","alias_value":"PCAKBK2U","created_at":"2026-07-05T09:53:53.214105+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.06385","citing_title":"Application-Driven Pedagogical Knowledge Optimization of Open-Source LLMs via Reinforcement Learning and Supervised Fine-Tuning","ref_index":5,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP","json":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP.json","graph_json":"https://pith.science/api/pith-number/PCAKBK2UR5BY3YVYOQTTYMPISP/graph.json","events_json":"https://pith.science/api/pith-number/PCAKBK2UR5BY3YVYOQTTYMPISP/events.json","paper":"https://pith.science/paper/PCAKBK2U"},"agent_actions":{"view_html":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP","download_json":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP.json","view_paper":"https://pith.science/paper/PCAKBK2U","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.18279&json=true","fetch_graph":"https://pith.science/api/pith-number/PCAKBK2UR5BY3YVYOQTTYMPISP/graph.json","fetch_events":"https://pith.science/api/pith-number/PCAKBK2UR5BY3YVYOQTTYMPISP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP/action/storage_attestation","attest_author":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP/action/author_attestation","sign_citation":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP/action/citation_signature","submit_replication":"https://pith.science/pith/PCAKBK2UR5BY3YVYOQTTYMPISP/action/replication_record"}},"created_at":"2026-07-05T09:53:53.214105+00:00","updated_at":"2026-07-05T09:53:53.214105+00:00"}