{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:X7FQ3DXEJ6LUXN33MSR2BOS2H7","short_pith_number":"pith:X7FQ3DXE","schema_version":"1.0","canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","source":{"kind":"arxiv","id":"2312.00267","version":3},"attestation_state":"computed","paper":{"title":"Sample Efficient Preference Alignment in LLMs via Active Exploration","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Barbara Engelhardt, Ilija Bogunovic, Jeff Schneider, Ojash Neopane, Stefano Ermon, Syrine Belakaria, Vikramjeet Das, Viraj Mehta, Willie Neiswanger, Yijia Dai","submitted_at":"2023-12-01T00:54:02Z","abstract_excerpt":"Preference-based feedback is important for many applications in machine learning where evaluation of a reward function is not feasible. Notable recent examples arise in preference alignment for large language models, including in reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO). For many applications of preference alignment, the cost of acquiring human feedback can be substantial. In this work, we take advantage of the fact that one can often choose contexts at which to obtain human feedback to most efficiently identify a good policy, and formalize the"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2312.00267","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"34e60341c1ee748dc003c50aaafd089b5762511c95a1268540b74dea7388fa68","abstract_canon_sha256":"638c3d96bfecde606324094d27acde41a54160dc7669433b4e5125cf9632302f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:35:30.492585Z","signature_b64":"KSwCh7nEhlENpKarzXz96O+sYzBlrn5kZDm+5CzLPuDb3sPnI6sD1Wof7DYuj7hw5OEhBC+nR1upD1MJK32yBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","last_reissued_at":"2026-07-05T10:35:30.492069Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:35:30.492069Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Sample Efficient Preference Alignment in LLMs via Active Exploration","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Barbara Engelhardt, Ilija Bogunovic, Jeff Schneider, Ojash Neopane, Stefano Ermon, Syrine Belakaria, Vikramjeet Das, Viraj Mehta, Willie Neiswanger, Yijia Dai","submitted_at":"2023-12-01T00:54:02Z","abstract_excerpt":"Preference-based feedback is important for many applications in machine learning where evaluation of a reward function is not feasible. Notable recent examples arise in preference alignment for large language models, including in reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO). For many applications of preference alignment, the cost of acquiring human feedback can be substantial. In this work, we take advantage of the fact that one can often choose contexts at which to obtain human feedback to most efficiently identify a good policy, and formalize the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.00267","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.00267/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2312.00267","created_at":"2026-07-05T10:35:30.492126+00:00"},{"alias_kind":"arxiv_version","alias_value":"2312.00267v3","created_at":"2026-07-05T10:35:30.492126+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.00267","created_at":"2026-07-05T10:35:30.492126+00:00"},{"alias_kind":"pith_short_12","alias_value":"X7FQ3DXEJ6LU","created_at":"2026-07-05T10:35:30.492126+00:00"},{"alias_kind":"pith_short_16","alias_value":"X7FQ3DXEJ6LUXN33","created_at":"2026-07-05T10:35:30.492126+00:00"},{"alias_kind":"pith_short_8","alias_value":"X7FQ3DXE","created_at":"2026-07-05T10:35:30.492126+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2505.19241","citing_title":"ActiveDPO: Active Direct Preference Optimization for Sample-Efficient Alignment","ref_index":17,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7","json":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7.json","graph_json":"https://pith.science/api/pith-number/X7FQ3DXEJ6LUXN33MSR2BOS2H7/graph.json","events_json":"https://pith.science/api/pith-number/X7FQ3DXEJ6LUXN33MSR2BOS2H7/events.json","paper":"https://pith.science/paper/X7FQ3DXE"},"agent_actions":{"view_html":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7","download_json":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7.json","view_paper":"https://pith.science/paper/X7FQ3DXE","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2312.00267&json=true","fetch_graph":"https://pith.science/api/pith-number/X7FQ3DXEJ6LUXN33MSR2BOS2H7/graph.json","fetch_events":"https://pith.science/api/pith-number/X7FQ3DXEJ6LUXN33MSR2BOS2H7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/action/storage_attestation","attest_author":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/action/author_attestation","sign_citation":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/action/citation_signature","submit_replication":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/action/replication_record"}},"created_at":"2026-07-05T10:35:30.492126+00:00","updated_at":"2026-07-05T10:35:30.492126+00:00"}