{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:DJ6ELZMMJUFB5ZLDJPX4S6SELX","short_pith_number":"pith:DJ6ELZMM","schema_version":"1.0","canonical_sha256":"1a7c45e58c4d0a1ee5634befc97a445de59ec49bd499c44667a23eb7b524eaa0","source":{"kind":"arxiv","id":"2009.01325","version":3},"attestation_state":"computed","paper":{"title":"Learning to summarize from human feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Alec Radford, Chelsea Voss, Daniel M. Ziegler, Dario Amodei, Jeff Wu, Long Ouyang, Nisan Stiennon, Paul Christiano, Ryan Lowe","submitted_at":"2020-09-02T19:54:41Z","abstract_excerpt":"As language models become more powerful, training and evaluation are increasingly bottlenecked by the data and metrics used for a particular task. For example, summarization models are often trained to predict human reference summaries and evaluated using ROUGE, but both of these metrics are rough proxies for what we really care about -- summary quality. In this work, we show that it is possible to significantly improve summary quality by training a model to optimize for human preferences. We collect a large, high-quality dataset of human comparisons between summaries, train a model to predict"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2009.01325","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-02T19:54:41Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"3627b39756a6f7cee97efa10f330ffbd3a6ae0b730e4c9f75daadbc2713d22ce","abstract_canon_sha256":"d95d58d71157c02b45df7556157c0442ae8cff37b423c35a2660340a50400e4a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T01:40:46.511579Z","signature_b64":"8fRwmf558LUtCoDTFBxQpm0nrRc23taEzVSioZR8qfBOU5GZ9C7VsR81yIAVkdoRXxk+ukRyOfTPH9YcSQROCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1a7c45e58c4d0a1ee5634befc97a445de59ec49bd499c44667a23eb7b524eaa0","last_reissued_at":"2026-05-18T01:40:46.510846Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T01:40:46.510846Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Learning to summarize from human feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Alec Radford, Chelsea Voss, Daniel M. Ziegler, Dario Amodei, Jeff Wu, Long Ouyang, Nisan Stiennon, Paul Christiano, Ryan Lowe","submitted_at":"2020-09-02T19:54:41Z","abstract_excerpt":"As language models become more powerful, training and evaluation are increasingly bottlenecked by the data and metrics used for a particular task. For example, summarization models are often trained to predict human reference summaries and evaluated using ROUGE, but both of these metrics are rough proxies for what we really care about -- summary quality. In this work, we show that it is possible to significantly improve summary quality by training a model to optimize for human preferences. We collect a large, high-quality dataset of human comparisons between summaries, train a model to predict"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.01325","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2009.01325","created_at":"2026-05-18T01:40:46.510965+00:00"},{"alias_kind":"arxiv_version","alias_value":"2009.01325v3","created_at":"2026-05-18T01:40:46.510965+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.01325","created_at":"2026-05-18T01:40:46.510965+00:00"},{"alias_kind":"pith_short_12","alias_value":"DJ6ELZMMJUFB","created_at":"2026-05-18T12:33:33.725879+00:00"},{"alias_kind":"pith_short_16","alias_value":"DJ6ELZMMJUFB5ZLD","created_at":"2026-05-18T12:33:33.725879+00:00"},{"alias_kind":"pith_short_8","alias_value":"DJ6ELZMM","created_at":"2026-05-18T12:33:33.725879+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":26,"internal_anchor_count":26,"sample":[{"citing_arxiv_id":"2204.06745","citing_title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","ref_index":89,"is_internal_anchor":true},{"citing_arxiv_id":"2504.10013","citing_title":"Training LLMs on HPC Systems: Best Practices from the OpenGPT-X Project","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2201.03544","citing_title":"The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20164","citing_title":"Not Every Rubric Teaches Equally: Policy-Aware Rubric Rewards for RLVR","ref_index":25,"is_internal_anchor":true},{"citing_arxiv_id":"2506.08125","citing_title":"Not All Tokens Matter: Towards Efficient LLM Reasoning via Token Significance in Reinforcement Learning","ref_index":31,"is_internal_anchor":true},{"citing_arxiv_id":"2310.06452","citing_title":"Understanding the Effects of RLHF on LLM Generalisation and Diversity","ref_index":4,"is_internal_anchor":true},{"citing_arxiv_id":"2505.15134","citing_title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","ref_index":78,"is_internal_anchor":true},{"citing_arxiv_id":"2509.20265","citing_title":"Failure Modes of Maximum Entropy RLHF","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2102.01293","citing_title":"Scaling Laws for Transfer","ref_index":192,"is_internal_anchor":true},{"citing_arxiv_id":"2207.14255","citing_title":"Efficient Training of Language Models to Fill in the Middle","ref_index":139,"is_internal_anchor":true},{"citing_arxiv_id":"2205.10487","citing_title":"Scaling Laws and Interpretability of Learning from Repeated Data","ref_index":20,"is_internal_anchor":true},{"citing_arxiv_id":"2308.03825","citing_title":"\"Do Anything Now\": Characterizing and Evaluating In-The-Wild Jailbreak Prompts on Large Language Models","ref_index":77,"is_internal_anchor":true},{"citing_arxiv_id":"2403.07691","citing_title":"ORPO: Monolithic Preference Optimization without Reference Model","ref_index":47,"is_internal_anchor":true},{"citing_arxiv_id":"2212.03827","citing_title":"Discovering Latent Knowledge in Language Models Without Supervision","ref_index":30,"is_internal_anchor":true},{"citing_arxiv_id":"2605.11533","citing_title":"Checkup2Action: A Multimodal Clinical Check-up Report Dataset for Patient-Oriented Action Card Generation","ref_index":23,"is_internal_anchor":true},{"citing_arxiv_id":"2302.12192","citing_title":"Aligning Text-to-Image Models using Human Feedback","ref_index":21,"is_internal_anchor":true},{"citing_arxiv_id":"2605.11533","citing_title":"Checkup2Action: A Multimodal Clinical Check-up Report Dataset for Patient-Oriented Action Card Generation","ref_index":23,"is_internal_anchor":true},{"citing_arxiv_id":"2605.05812","citing_title":"Long-Horizon Q-Learning: Accurate Value Learning via n-Step Inequalities","ref_index":40,"is_internal_anchor":true},{"citing_arxiv_id":"2605.05812","citing_title":"Long-Horizon Q-Learning: Accurate Value Learning via n-Step Inequalities","ref_index":40,"is_internal_anchor":true},{"citing_arxiv_id":"2605.05481","citing_title":"Approximate Next Policy Sampling: Replacing Conservative Target Policy Updates in Deep RL","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2605.01311","citing_title":"The Partial Testimony of Logs: Evaluation of Language Model Generation under Confounded Model Choice","ref_index":29,"is_internal_anchor":true},{"citing_arxiv_id":"2112.00861","citing_title":"A General Language Assistant as a Laboratory for Alignment","ref_index":248,"is_internal_anchor":true},{"citing_arxiv_id":"2604.09813","citing_title":"Controllable and Verifiable Tool-Use Data Synthesis for Agentic Reinforcement Learning","ref_index":21,"is_internal_anchor":true},{"citing_arxiv_id":"2303.18223","citing_title":"A Survey of Large Language Models","ref_index":131,"is_internal_anchor":true},{"citing_arxiv_id":"2604.16995","citing_title":"SPS: Steering Probability Squeezing for Better Exploration in Reinforcement Learning for Large Language Models","ref_index":28,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX","json":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX.json","graph_json":"https://pith.science/api/pith-number/DJ6ELZMMJUFB5ZLDJPX4S6SELX/graph.json","events_json":"https://pith.science/api/pith-number/DJ6ELZMMJUFB5ZLDJPX4S6SELX/events.json","paper":"https://pith.science/paper/DJ6ELZMM"},"agent_actions":{"view_html":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX","download_json":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX.json","view_paper":"https://pith.science/paper/DJ6ELZMM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2009.01325&json=true","fetch_graph":"https://pith.science/api/pith-number/DJ6ELZMMJUFB5ZLDJPX4S6SELX/graph.json","fetch_events":"https://pith.science/api/pith-number/DJ6ELZMMJUFB5ZLDJPX4S6SELX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX/action/storage_attestation","attest_author":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX/action/author_attestation","sign_citation":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX/action/citation_signature","submit_replication":"https://pith.science/pith/DJ6ELZMMJUFB5ZLDJPX4S6SELX/action/replication_record"}},"created_at":"2026-05-18T01:40:46.510965+00:00","updated_at":"2026-05-18T01:40:46.510965+00:00"}