{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:TF3JR2KMBRLJWM2DQYWGFF7LHX","short_pith_number":"pith:TF3JR2KM","canonical_record":{"source":{"id":"2410.02743","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","cross_cats_sorted":[],"title_canon_sha256":"a6e98eebfc3dc407ecb9ee2519c3f2d46db308441a0d82e8feb6ccdb7c5e7f74","abstract_canon_sha256":"997e0f8bfd799f1bc8ce4df6b5476e634b4dcc2867cee78cbc8dc2666fffc3e7"},"schema_version":"1.0"},"canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","source":{"kind":"arxiv","id":"2410.02743","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.02743","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"arxiv_version","alias_value":"2410.02743v2","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.02743","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_12","alias_value":"TF3JR2KMBRLJ","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_16","alias_value":"TF3JR2KMBRLJWM2D","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_8","alias_value":"TF3JR2KM","created_at":"2026-07-05T10:14:46Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:TF3JR2KMBRLJWM2DQYWGFF7LHX","target":"record","payload":{"canonical_record":{"source":{"id":"2410.02743","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","cross_cats_sorted":[],"title_canon_sha256":"a6e98eebfc3dc407ecb9ee2519c3f2d46db308441a0d82e8feb6ccdb7c5e7f74","abstract_canon_sha256":"997e0f8bfd799f1bc8ce4df6b5476e634b4dcc2867cee78cbc8dc2666fffc3e7"},"schema_version":"1.0"},"canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:14:46.074389Z","signature_b64":"reS0WbV2xv/0hOJcuGg6/fLaatHZ50RGZyFf0G5LT4bAhMhW30EYxNxgseMz6SrVjkvr5pZeeVN1jl6J1NQRCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","last_reissued_at":"2026-07-05T10:14:46.073868Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:14:46.073868Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.02743","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:14:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dPd0D9dqS5keu/LRg/1sAsn8TWj22MYdfuxqAr10cGtbTMd64fYJsBESRF3p6UbdF8H0PrzD2p43m9ACqkbGDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:48:55.183376Z"},"content_sha256":"fe4d2040bd70853fee75c1159edd7226511e320d0084d9e7c77f6f8846011d63","schema_version":"1.0","event_id":"sha256:fe4d2040bd70853fee75c1159edd7226511e320d0084d9e7c77f6f8846011d63"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:TF3JR2KMBRLJWM2DQYWGFF7LHX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Haoran Sun, Huang Fang, Hua Wu, Shuohuan Wang, Yekun Chai, Yu Sun","submitted_at":"2024-10-03T17:55:13Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated effectiveness in aligning large language models (LLMs) with human preferences. However, token-level RLHF suffers from the credit assignment problem over long sequences, where delayed rewards make it challenging for the model to discern which actions contributed to preferred outcomes. This hinders learning efficiency and slows convergence.In this paper, we propose MA-RLHF, a simple yet effective RLHF framework that incorporates macro actions -- sequences of tokens or higher-level language constructs -- into the learning process."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.02743","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.02743/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:14:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TRmNxKmrYwLESopHj03ewztoxzmCXeQ2Chhtm8lc7/53vveD+AS7e4Ycgzg0QBd/EH03A8GkGDXLpGP54c9nBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:48:55.184122Z"},"content_sha256":"5cedfdd256626aeb6083c9b27c19a42c527888cb236b901fae8ee08ad933bac2","schema_version":"1.0","event_id":"sha256:5cedfdd256626aeb6083c9b27c19a42c527888cb236b901fae8ee08ad933bac2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/bundle.json","state_url":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T05:48:55Z","links":{"resolver":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX","bundle":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/bundle.json","state":"https://pith.science/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TF3JR2KMBRLJWM2DQYWGFF7LHX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TF3JR2KMBRLJWM2DQYWGFF7LHX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"997e0f8bfd799f1bc8ce4df6b5476e634b4dcc2867cee78cbc8dc2666fffc3e7","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","title_canon_sha256":"a6e98eebfc3dc407ecb9ee2519c3f2d46db308441a0d82e8feb6ccdb7c5e7f74"},"schema_version":"1.0","source":{"id":"2410.02743","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.02743","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"arxiv_version","alias_value":"2410.02743v2","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.02743","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_12","alias_value":"TF3JR2KMBRLJ","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_16","alias_value":"TF3JR2KMBRLJWM2D","created_at":"2026-07-05T10:14:46Z"},{"alias_kind":"pith_short_8","alias_value":"TF3JR2KM","created_at":"2026-07-05T10:14:46Z"}],"graph_snapshots":[{"event_id":"sha256:5cedfdd256626aeb6083c9b27c19a42c527888cb236b901fae8ee08ad933bac2","target":"graph","created_at":"2026-07-05T10:14:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.02743/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated effectiveness in aligning large language models (LLMs) with human preferences. However, token-level RLHF suffers from the credit assignment problem over long sequences, where delayed rewards make it challenging for the model to discern which actions contributed to preferred outcomes. This hinders learning efficiency and slows convergence.In this paper, we propose MA-RLHF, a simple yet effective RLHF framework that incorporates macro actions -- sequences of tokens or higher-level language constructs -- into the learning process.","authors_text":"Haoran Sun, Huang Fang, Hua Wu, Shuohuan Wang, Yekun Chai, Yu Sun","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.02743","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fe4d2040bd70853fee75c1159edd7226511e320d0084d9e7c77f6f8846011d63","target":"record","created_at":"2026-07-05T10:14:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"997e0f8bfd799f1bc8ce4df6b5476e634b4dcc2867cee78cbc8dc2666fffc3e7","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-03T17:55:13Z","title_canon_sha256":"a6e98eebfc3dc407ecb9ee2519c3f2d46db308441a0d82e8feb6ccdb7c5e7f74"},"schema_version":"1.0","source":{"id":"2410.02743","kind":"arxiv","version":2}},"canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"997698e94c0c569b3343862c6297eb3de217a21ad16cd538b56aba2e568b228c","first_computed_at":"2026-07-05T10:14:46.073868Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:14:46.073868Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"reS0WbV2xv/0hOJcuGg6/fLaatHZ50RGZyFf0G5LT4bAhMhW30EYxNxgseMz6SrVjkvr5pZeeVN1jl6J1NQRCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:14:46.074389Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.02743","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fe4d2040bd70853fee75c1159edd7226511e320d0084d9e7c77f6f8846011d63","sha256:5cedfdd256626aeb6083c9b27c19a42c527888cb236b901fae8ee08ad933bac2"],"state_sha256":"6fb54078cc7e7ef0f70dd9ddfcd1a818a2c8275ee876cb35d872b106cc975030"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CJAR8Phx7o5P5E3HAz15lGniGpTtLDmLjZKvQ5Zkwbz1vDbPLhs/KH7U5hdrdZfPIxC1rdke/DV73CtzlRG5Cg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T05:48:55.188553Z","bundle_sha256":"0e97967f08235300d255271b0ad40b96b3d09e540056e85d1c031e18572e827a"}}