{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:CA3OVUO52HWOEEG3G2ICLZ22I4","short_pith_number":"pith:CA3OVUO5","schema_version":"1.0","canonical_sha256":"1036ead1ddd1ece210db369025e75a4708c18c603d48c4fae7df7cdaf67862b0","source":{"kind":"arxiv","id":"2310.03716","version":2},"attestation_state":"computed","paper":{"title":"A Long Way to Go: Investigating Length Correlations in RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Greg Durrett, Jiacheng Xu, Prasann Singhal, Tanya Goyal","submitted_at":"2023-10-05T17:38:28Z","abstract_excerpt":"Great success has been reported using Reinforcement Learning from Human Feedback (RLHF) to align large language models, with open preference datasets enabling wider experimentation, particularly for \"helpfulness\" in tasks like dialogue and web question answering. Alongside these improvements, however, RLHF also often drives models to produce longer outputs. This paper demonstrates, on three diverse settings, that optimizing for response length is, much more than previously thought, a significant factor behind RLHF. Studying the strategies RL optimization uses to maximize reward, we find improv"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.03716","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-05T17:38:28Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2b266e063771a60007e136e98226781d71c5cf54ac23b02f18aeca2b94e25d04","abstract_canon_sha256":"dcec4162a9a147be5d44e8d3cd088a0276b48b5194092d57749b23f23c89390e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:42:30.485330Z","signature_b64":"dKVhT11HQL3JFT92mJmc/otiLiOttLDfbxVjoeGIXbolQxwOxIkDxjELbyqr2eL94Pi5bXWzVEfrXwXKbDy6Bw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1036ead1ddd1ece210db369025e75a4708c18c603d48c4fae7df7cdaf67862b0","last_reissued_at":"2026-07-05T08:42:30.484852Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:42:30.484852Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Long Way to Go: Investigating Length Correlations in RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Greg Durrett, Jiacheng Xu, Prasann Singhal, Tanya Goyal","submitted_at":"2023-10-05T17:38:28Z","abstract_excerpt":"Great success has been reported using Reinforcement Learning from Human Feedback (RLHF) to align large language models, with open preference datasets enabling wider experimentation, particularly for \"helpfulness\" in tasks like dialogue and web question answering. Alongside these improvements, however, RLHF also often drives models to produce longer outputs. This paper demonstrates, on three diverse settings, that optimizing for response length is, much more than previously thought, a significant factor behind RLHF. Studying the strategies RL optimization uses to maximize reward, we find improv"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.03716","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.03716/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.03716","created_at":"2026-07-05T08:42:30.484910+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.03716v2","created_at":"2026-07-05T08:42:30.484910+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.03716","created_at":"2026-07-05T08:42:30.484910+00:00"},{"alias_kind":"pith_short_12","alias_value":"CA3OVUO52HWO","created_at":"2026-07-05T08:42:30.484910+00:00"},{"alias_kind":"pith_short_16","alias_value":"CA3OVUO52HWOEEG3","created_at":"2026-07-05T08:42:30.484910+00:00"},{"alias_kind":"pith_short_8","alias_value":"CA3OVUO5","created_at":"2026-07-05T08:42:30.484910+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":34,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.05904","citing_title":"More Convincing, Not More Correct: Self-Play Reward Hacking of Reference-Free LLM Judges","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2606.25432","citing_title":"Brevity is the Soul of Inference Efficiency: Inducing Concision in VLMs via Data Curation","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2606.22329","citing_title":"BabelJudge: Measuring LLM-as-a-Judge Reliability Across Languages and Agent Trajectories","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21732","citing_title":"Safe to Check, Unsafe to Use: Relinking at the Compression Boundary of LLM Agents","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2606.19057","citing_title":"Quantifying and Auditing LLM Evaluation via Positive--Unlabeled Learning","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11635","citing_title":"Are LLMs Bad at Moral Reasoning?","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09711","citing_title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","ref_index":263,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09073","citing_title":"A Unifying Lens on Reward Uncertainty in RLHF","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07988","citing_title":"PAFO: Pareto Fairness Optimization for Personalized Reward Modeling","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04781","citing_title":"AIP: A Graph Representation for Learning and Governing Agent Skills","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04273","citing_title":"Human agency in initial human-AI proof formalization workflows","ref_index":297,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04075","citing_title":"Large Language Models Hack Rewards, and Society","ref_index":52,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03131","citing_title":"HARVE: Hacking-Aware Reward-Head Vector Editing for Robust Reward Models","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2606.25432","citing_title":"Brevity is the Soul of Inference Efficiency: Inducing Concision in VLMs via Data Curation","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30844","citing_title":"Fine-Tuning Improves Information Conveyance in Language Models","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05054","citing_title":"Boosting Self-Consistency with Ranking","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18721","citing_title":"General Preference Reinforcement Learning","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2507.15698","citing_title":"CoLD: Counterfactually-Guided Length Debiasing for Process Reward Models in Mathematical Reasoning","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2601.21350","citing_title":"Factored Causal Representation Learning for Robust Reward Modeling in RLHF","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18721","citing_title":"General Preference Reinforcement Learning","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2605.16339","citing_title":"Preference Instability in Reward Models: Detection and Mitigation via Sparse Autoencoders","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18721","citing_title":"General Preference Reinforcement Learning","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15207","citing_title":"TeamTR: Trust-Region Fine-Tuning for Multi-Agent LLM Coordination","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2503.04697","citing_title":"L1: Controlling How Long A Reasoning Model Thinks With Reinforcement Learning","ref_index":16,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4","json":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4.json","graph_json":"https://pith.science/api/pith-number/CA3OVUO52HWOEEG3G2ICLZ22I4/graph.json","events_json":"https://pith.science/api/pith-number/CA3OVUO52HWOEEG3G2ICLZ22I4/events.json","paper":"https://pith.science/paper/CA3OVUO5"},"agent_actions":{"view_html":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4","download_json":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4.json","view_paper":"https://pith.science/paper/CA3OVUO5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.03716&json=true","fetch_graph":"https://pith.science/api/pith-number/CA3OVUO52HWOEEG3G2ICLZ22I4/graph.json","fetch_events":"https://pith.science/api/pith-number/CA3OVUO52HWOEEG3G2ICLZ22I4/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4/action/storage_attestation","attest_author":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4/action/author_attestation","sign_citation":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4/action/citation_signature","submit_replication":"https://pith.science/pith/CA3OVUO52HWOEEG3G2ICLZ22I4/action/replication_record"}},"created_at":"2026-07-05T08:42:30.484910+00:00","updated_at":"2026-07-05T08:42:30.484910+00:00"}