{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:TO5COMFGKGOKVVTUVS6MMJNJCR","short_pith_number":"pith:TO5COMFG","schema_version":"1.0","canonical_sha256":"9bba2730a6519caad674acbcc625a9144282c8f301a290549446b480b4358870","source":{"kind":"arxiv","id":"2310.05199","version":5},"attestation_state":"computed","paper":{"title":"Loose lips sink ships: Mitigating Length Bias in Reinforcement Learning from Human Feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jun Zhao, Qi Zhang, Rui Zheng, Shihan Dou, Tao Gui, Wei Shen, Wenyu Zhan, Xuanjing Huang","submitted_at":"2023-10-08T15:14:39Z","abstract_excerpt":"Reinforcement learning from human feedback serves as a crucial bridge, aligning large language models with human and societal values. This alignment requires a vast corpus of human feedback to learn a reward model, which is subsequently used to finetune language models. However, we have identified that the reward model often finds shortcuts to bypass its intended objectives, misleadingly assuming that humans prefer longer responses. The emergence of length bias often induces the model to favor longer outputs, yet it doesn't equate to an increase in helpful information within these outputs. In "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.05199","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-08T15:14:39Z","cross_cats_sorted":[],"title_canon_sha256":"7a0257446cb32dddc8b4ba5fcc596c34b56661e299cf235585779fd65c8f0718","abstract_canon_sha256":"8bcfe8991635bed17e8e8d63f93e85d3be5c40604fa131ae99ff5bfea0511729"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:18:12.117576Z","signature_b64":"1M2G0NqSTfm26/Q858S7LER3my5V+7BgjWKOiBwAutedQmTeK81VA4Lvdc6UzDk9vTKP4nqOgT62mW/+f+/0DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9bba2730a6519caad674acbcc625a9144282c8f301a290549446b480b4358870","last_reissued_at":"2026-07-05T07:18:12.117002Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:18:12.117002Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Loose lips sink ships: Mitigating Length Bias in Reinforcement Learning from Human Feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jun Zhao, Qi Zhang, Rui Zheng, Shihan Dou, Tao Gui, Wei Shen, Wenyu Zhan, Xuanjing Huang","submitted_at":"2023-10-08T15:14:39Z","abstract_excerpt":"Reinforcement learning from human feedback serves as a crucial bridge, aligning large language models with human and societal values. This alignment requires a vast corpus of human feedback to learn a reward model, which is subsequently used to finetune language models. However, we have identified that the reward model often finds shortcuts to bypass its intended objectives, misleadingly assuming that humans prefer longer responses. The emergence of length bias often induces the model to favor longer outputs, yet it doesn't equate to an increase in helpful information within these outputs. In "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.05199","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.05199/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.05199","created_at":"2026-07-05T07:18:12.117063+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.05199v5","created_at":"2026-07-05T07:18:12.117063+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.05199","created_at":"2026-07-05T07:18:12.117063+00:00"},{"alias_kind":"pith_short_12","alias_value":"TO5COMFGKGOK","created_at":"2026-07-05T07:18:12.117063+00:00"},{"alias_kind":"pith_short_16","alias_value":"TO5COMFGKGOKVVTU","created_at":"2026-07-05T07:18:12.117063+00:00"},{"alias_kind":"pith_short_8","alias_value":"TO5COMFG","created_at":"2026-07-05T07:18:12.117063+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.09043","citing_title":"DynaCF: Mitigating Shortcut Learning in Reward Models via Dynamic Counterfactual Sensitivity","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2507.15698","citing_title":"CoLD: Counterfactually-Guided Length Debiasing for Process Reward Models in Mathematical Reasoning","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2506.12382","citing_title":"Exploring the Secondary Risks of Large Language Models","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2507.06419","citing_title":"Teach a Reward Model to Correct Itself: Reward Guided Adversarial Failure Discovery for Robust Reward Modeling","ref_index":29,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR","json":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR.json","graph_json":"https://pith.science/api/pith-number/TO5COMFGKGOKVVTUVS6MMJNJCR/graph.json","events_json":"https://pith.science/api/pith-number/TO5COMFGKGOKVVTUVS6MMJNJCR/events.json","paper":"https://pith.science/paper/TO5COMFG"},"agent_actions":{"view_html":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR","download_json":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR.json","view_paper":"https://pith.science/paper/TO5COMFG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.05199&json=true","fetch_graph":"https://pith.science/api/pith-number/TO5COMFGKGOKVVTUVS6MMJNJCR/graph.json","fetch_events":"https://pith.science/api/pith-number/TO5COMFGKGOKVVTUVS6MMJNJCR/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR/action/storage_attestation","attest_author":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR/action/author_attestation","sign_citation":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR/action/citation_signature","submit_replication":"https://pith.science/pith/TO5COMFGKGOKVVTUVS6MMJNJCR/action/replication_record"}},"created_at":"2026-07-05T07:18:12.117063+00:00","updated_at":"2026-07-05T07:18:12.117063+00:00"}