{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:RKGJ3WNHDE45LTF7VLG25GC7YJ","short_pith_number":"pith:RKGJ3WNH","schema_version":"1.0","canonical_sha256":"8a8c9dd9a71939d5ccbfaacdae985fc248e9e137660b07d9cd67fd1798113fd7","source":{"kind":"arxiv","id":"2310.13639","version":3},"attestation_state":"computed","paper":{"title":"Contrastive Preference Learning: Learning from Human Feedback without RL","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Chelsea Finn, Dorsa Sadigh, Harshit Sikchi, Joey Hejna, Rafael Rafailov, Scott Niekum, W. Bradley Knox","submitted_at":"2023-10-20T16:37:56Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has emerged as a popular paradigm for aligning models with human intent. Typically RLHF algorithms operate in two phases: first, use human preferences to learn a reward function and second, align the model by optimizing the learned reward via reinforcement learning (RL). This paradigm assumes that human preferences are distributed according to reward, but recent work suggests that they instead follow the regret under the user's optimal policy. Thus, learning a reward function from feedback is not only based on a flawed assumption of human prefe"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.13639","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-20T16:37:56Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"16cb84ae9cde7a464c42ae78835b7346e841d764b28b8d9ee95553caa595b678","abstract_canon_sha256":"13927a2a82f98c237e2718b9ecddd3b1209d93418d89e5ac0cda2381bf512ac1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:13:27.752246Z","signature_b64":"kjajFAtCDZdPQ1L13dNYb+9Q62pVgrFVR378JTOkGcprF9ai8vxUOdz4DkCLPaITPcbQVABasApIvzdaTqfqBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8a8c9dd9a71939d5ccbfaacdae985fc248e9e137660b07d9cd67fd1798113fd7","last_reissued_at":"2026-07-05T08:13:27.751798Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:13:27.751798Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Contrastive Preference Learning: Learning from Human Feedback without RL","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Chelsea Finn, Dorsa Sadigh, Harshit Sikchi, Joey Hejna, Rafael Rafailov, Scott Niekum, W. Bradley Knox","submitted_at":"2023-10-20T16:37:56Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has emerged as a popular paradigm for aligning models with human intent. Typically RLHF algorithms operate in two phases: first, use human preferences to learn a reward function and second, align the model by optimizing the learned reward via reinforcement learning (RL). This paradigm assumes that human preferences are distributed according to reward, but recent work suggests that they instead follow the regret under the user's optimal policy. Thus, learning a reward function from feedback is not only based on a flawed assumption of human prefe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.13639","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.13639/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.13639","created_at":"2026-07-05T08:13:27.751856+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.13639v3","created_at":"2026-07-05T08:13:27.751856+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.13639","created_at":"2026-07-05T08:13:27.751856+00:00"},{"alias_kind":"pith_short_12","alias_value":"RKGJ3WNHDE45","created_at":"2026-07-05T08:13:27.751856+00:00"},{"alias_kind":"pith_short_16","alias_value":"RKGJ3WNHDE45LTF7","created_at":"2026-07-05T08:13:27.751856+00:00"},{"alias_kind":"pith_short_8","alias_value":"RKGJ3WNH","created_at":"2026-07-05T08:13:27.751856+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.12372","citing_title":"UniIntervene: Agentic Intervention for Efficient Real-World Reinforcement Learning","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09124","citing_title":"A Regret Minimization Framework on Preference Learning in Large Language Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.18703","citing_title":"Contextualizing Biological Language Models across Modalities via Logit-Space Contrastive Alignment","ref_index":94,"is_internal_anchor":false},{"citing_arxiv_id":"2411.00361","citing_title":"Direct Preference Optimization for Primitive-Enabled Hierarchical RL: A Bilevel Approach","ref_index":16,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ","json":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ.json","graph_json":"https://pith.science/api/pith-number/RKGJ3WNHDE45LTF7VLG25GC7YJ/graph.json","events_json":"https://pith.science/api/pith-number/RKGJ3WNHDE45LTF7VLG25GC7YJ/events.json","paper":"https://pith.science/paper/RKGJ3WNH"},"agent_actions":{"view_html":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ","download_json":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ.json","view_paper":"https://pith.science/paper/RKGJ3WNH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.13639&json=true","fetch_graph":"https://pith.science/api/pith-number/RKGJ3WNHDE45LTF7VLG25GC7YJ/graph.json","fetch_events":"https://pith.science/api/pith-number/RKGJ3WNHDE45LTF7VLG25GC7YJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ/action/storage_attestation","attest_author":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ/action/author_attestation","sign_citation":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ/action/citation_signature","submit_replication":"https://pith.science/pith/RKGJ3WNHDE45LTF7VLG25GC7YJ/action/replication_record"}},"created_at":"2026-07-05T08:13:27.751856+00:00","updated_at":"2026-07-05T08:13:27.751856+00:00"}