{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:TF3JR2KMBRLJWM2DQYWGFF7LHX","short_pith_number":"pith:TF3JR2KM","schema_version":"1.0","canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","source":{"kind":"arxiv","id":"2410.02743","version":2},"attestation_state":"computed","paper":{"title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Haoran Sun, Huang Fang, Hua Wu, Shuohuan Wang, Yekun Chai, Yu Sun","submitted_at":"2024-10-03T17:55:13Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated effectiveness in aligning large language models (LLMs) with human preferences. However, token-level RLHF suffers from the credit assignment problem over long sequences, where delayed rewards make it challenging for the model to discern which actions contributed to preferred outcomes. This hinders learning efficiency and slows convergence.In this paper, we propose MA-RLHF, a simple yet effective RLHF framework that incorporates macro actions -- sequences of tokens or higher-level language constructs -- into the learning process."},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.02743","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","cross_cats_sorted":[],"title_canon_sha256":"a6e98eebfc3dc407ecb9ee2519c3f2d46db308441a0d82e8feb6ccdb7c5e7f74","abstract_canon_sha256":"997e0f8bfd799f1bc8ce4df6b5476e634b4dcc2867cee78cbc8dc2666fffc3e7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:14:46.074389Z","signature_b64":"reS0WbV2xv/0hOJcuGg6/fLaatHZ50RGZyFf0G5LT4bAhMhW30EYxNxgseMz6SrVjkvr5pZeeVN1jl6J1NQRCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","last_reissued_at":"2026-07-05T10:14:46.073868Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:14:46.073868Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Haoran Sun, Huang Fang, Hua Wu, Shuohuan Wang, Yekun Chai, Yu Sun","submitted_at":"2024-10-03T17:55:13Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated effectiveness in aligning large language models (LLMs) with human preferences. However, token-level RLHF suffers from the credit assignment problem over long sequences, where delayed rewards make it challenging for the model to discern which actions contributed to preferred outcomes. This hinders learning efficiency and slows convergence.In this paper, we propose MA-RLHF, a simple yet effective RLHF framework that incorporates macro actions -- sequences of tokens or higher-level language constructs -- into the learning process."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.02743","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.02743/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.02743","created_at":"2026-07-05T10:14:46.073926+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.02743v2","created_at":"2026-07-05T10:14:46.073926+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.02743","created_at":"2026-07-05T10:14:46.073926+00:00"},{"alias_kind":"pith_short_12","alias_value":"TF3JR2KMBRLJ","created_at":"2026-07-05T10:14:46.073926+00:00"},{"alias_kind":"pith_short_16","alias_value":"TF3JR2KMBRLJWM2D","created_at":"2026-07-05T10:14:46.073926+00:00"},{"alias_kind":"pith_short_8","alias_value":"TF3JR2KM","created_at":"2026-07-05T10:14:46.073926+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.17967","citing_title":"Reconciling Contradictory Views on the Effectiveness of SFT in LLMs: An Interaction Perspective","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX","json":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX.json","graph_json":"https://pith.science/api/pith-number/TF3JR2KMBRLJWM2DQYWGFF7LHX/graph.json","events_json":"https://pith.science/api/pith-number/TF3JR2KMBRLJWM2DQYWGFF7LHX/events.json","paper":"https://pith.science/paper/TF3JR2KM"},"agent_actions":{"view_html":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX","download_json":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX.json","view_paper":"https://pith.science/paper/TF3JR2KM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.02743&json=true","fetch_graph":"https://pith.science/api/pith-number/TF3JR2KMBRLJWM2DQYWGFF7LHX/graph.json","fetch_events":"https://pith.science/api/pith-number/TF3JR2KMBRLJWM2DQYWGFF7LHX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/action/storage_attestation","attest_author":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/action/author_attestation","sign_citation":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/action/citation_signature","submit_replication":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/action/replication_record"}},"created_at":"2026-07-05T10:14:46.073926+00:00","updated_at":"2026-07-05T10:14:46.073926+00:00"}