{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RQ2AC355JH2IR2HF7YOGOL6FCS","short_pith_number":"pith:RQ2AC355","schema_version":"1.0","canonical_sha256":"8c34016fbd49f488e8e5fe1c672fc5148db70033f17fbe9b31bde71a9e217e60","source":{"kind":"arxiv","id":"2405.14655","version":2},"attestation_state":"computed","paper":{"title":"Multi-turn Reinforcement Learning from Preference Human Feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Asaf Cassel, Avinatan Hassidim, Avital Zipori, Aviv Rosenberg, Bilal Piot, Daniele Calandriello, Hila Noga, Idan Szpektor, Lior Shani, Oran Lang, Orgad Keller, R\\'emi Munos, Yossi Matias","submitted_at":"2024-05-23T14:53:54Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has become the standard approach for aligning Large Language Models (LLMs) with human preferences, allowing LLMs to demonstrate remarkable abilities in various tasks. Existing methods work by emulating the preferences at the single decision (turn) level, limiting their capabilities in settings that require planning or multi-turn interactions to achieve a long-term goal. In this paper, we address this issue by developing novel methods for Reinforcement Learning (RL) from preference feedback between two full multi-turn conversations. In the tabul"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.14655","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-23T14:53:54Z","cross_cats_sorted":[],"title_canon_sha256":"2b85a3456dff451dd108feb3442e5f9f1567ff161086a03923b9afde920b263b","abstract_canon_sha256":"8abb2d05e5af6a010b8e7633cb06f477d5d33a8cd4891e51ba3d1b9d31d8aff3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:43:07.282422Z","signature_b64":"tfzBhaGVIV8HDAOeI8YcoZtv1Zi4vQPgJaPAVBuDvQQnb+Iu8gihL6Tp/9wA5h4xn8hx5cxhOd+/3+wH9cunBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8c34016fbd49f488e8e5fe1c672fc5148db70033f17fbe9b31bde71a9e217e60","last_reissued_at":"2026-07-05T09:43:07.280550Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:43:07.280550Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Multi-turn Reinforcement Learning from Preference Human Feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Asaf Cassel, Avinatan Hassidim, Avital Zipori, Aviv Rosenberg, Bilal Piot, Daniele Calandriello, Hila Noga, Idan Szpektor, Lior Shani, Oran Lang, Orgad Keller, R\\'emi Munos, Yossi Matias","submitted_at":"2024-05-23T14:53:54Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has become the standard approach for aligning Large Language Models (LLMs) with human preferences, allowing LLMs to demonstrate remarkable abilities in various tasks. Existing methods work by emulating the preferences at the single decision (turn) level, limiting their capabilities in settings that require planning or multi-turn interactions to achieve a long-term goal. In this paper, we address this issue by developing novel methods for Reinforcement Learning (RL) from preference feedback between two full multi-turn conversations. In the tabul"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.14655","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.14655/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.14655","created_at":"2026-07-05T09:43:07.281944+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.14655v2","created_at":"2026-07-05T09:43:07.281944+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.14655","created_at":"2026-07-05T09:43:07.281944+00:00"},{"alias_kind":"pith_short_12","alias_value":"RQ2AC355JH2I","created_at":"2026-07-05T09:43:07.281944+00:00"},{"alias_kind":"pith_short_16","alias_value":"RQ2AC355JH2IR2HF","created_at":"2026-07-05T09:43:07.281944+00:00"},{"alias_kind":"pith_short_8","alias_value":"RQ2AC355","created_at":"2026-07-05T09:43:07.281944+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2409.12917","citing_title":"Training Language Models to Self-Correct via Reinforcement Learning","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2501.09686","citing_title":"Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models","ref_index":128,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS","json":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS.json","graph_json":"https://pith.science/api/pith-number/RQ2AC355JH2IR2HF7YOGOL6FCS/graph.json","events_json":"https://pith.science/api/pith-number/RQ2AC355JH2IR2HF7YOGOL6FCS/events.json","paper":"https://pith.science/paper/RQ2AC355"},"agent_actions":{"view_html":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS","download_json":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS.json","view_paper":"https://pith.science/paper/RQ2AC355","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.14655&json=true","fetch_graph":"https://pith.science/api/pith-number/RQ2AC355JH2IR2HF7YOGOL6FCS/graph.json","fetch_events":"https://pith.science/api/pith-number/RQ2AC355JH2IR2HF7YOGOL6FCS/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS/action/storage_attestation","attest_author":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS/action/author_attestation","sign_citation":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS/action/citation_signature","submit_replication":"https://pith.science/pith/RQ2AC355JH2IR2HF7YOGOL6FCS/action/replication_record"}},"created_at":"2026-07-05T09:43:07.281944+00:00","updated_at":"2026-07-05T09:43:07.281944+00:00"}