{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:X7FQ3DXEJ6LUXN33MSR2BOS2H7","short_pith_number":"pith:X7FQ3DXE","canonical_record":{"source":{"id":"2312.00267","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"34e60341c1ee748dc003c50aaafd089b5762511c95a1268540b74dea7388fa68","abstract_canon_sha256":"638c3d96bfecde606324094d27acde41a54160dc7669433b4e5125cf9632302f"},"schema_version":"1.0"},"canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","source":{"kind":"arxiv","id":"2312.00267","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.00267","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"arxiv_version","alias_value":"2312.00267v3","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.00267","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_12","alias_value":"X7FQ3DXEJ6LU","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_16","alias_value":"X7FQ3DXEJ6LUXN33","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_8","alias_value":"X7FQ3DXE","created_at":"2026-07-05T10:35:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:X7FQ3DXEJ6LUXN33MSR2BOS2H7","target":"record","payload":{"canonical_record":{"source":{"id":"2312.00267","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"34e60341c1ee748dc003c50aaafd089b5762511c95a1268540b74dea7388fa68","abstract_canon_sha256":"638c3d96bfecde606324094d27acde41a54160dc7669433b4e5125cf9632302f"},"schema_version":"1.0"},"canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:35:30.492585Z","signature_b64":"KSwCh7nEhlENpKarzXz96O+sYzBlrn5kZDm+5CzLPuDb3sPnI6sD1Wof7DYuj7hw5OEhBC+nR1upD1MJK32yBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","last_reissued_at":"2026-07-05T10:35:30.492069Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:35:30.492069Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.00267","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:35:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SAQ+3HcDIN/BMxopv8y/Hqt7f1lOc/ILaAXXQWSBoUkGD4nO0YuTk5u8i/RqZw5Ni2U/W1Rna0/qAG41/cdBDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T13:09:00.510948Z"},"content_sha256":"5b1d2e9df2823ac9ac014a9b3fefc71def35529f0a0b717fb1c6206565869b91","schema_version":"1.0","event_id":"sha256:5b1d2e9df2823ac9ac014a9b3fefc71def35529f0a0b717fb1c6206565869b91"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:X7FQ3DXEJ6LUXN33MSR2BOS2H7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Sample Efficient Preference Alignment in LLMs via Active Exploration","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Barbara Engelhardt, Ilija Bogunovic, Jeff Schneider, Ojash Neopane, Stefano Ermon, Syrine Belakaria, Vikramjeet Das, Viraj Mehta, Willie Neiswanger, Yijia Dai","submitted_at":"2023-12-01T00:54:02Z","abstract_excerpt":"Preference-based feedback is important for many applications in machine learning where evaluation of a reward function is not feasible. Notable recent examples arise in preference alignment for large language models, including in reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO). For many applications of preference alignment, the cost of acquiring human feedback can be substantial. In this work, we take advantage of the fact that one can often choose contexts at which to obtain human feedback to most efficiently identify a good policy, and formalize the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.00267","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.00267/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:35:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"a/PMue0nh3d8f2ZcaVqYskQb42ab+zFgpESmjNCPHXLMNDNnWDW3TGJTvyAKtOjCCMvNod87aQp3lO5gxOJOAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T13:09:00.511724Z"},"content_sha256":"de29953b6a0698482c6a4dca73877ac6d133c8e0d9be1ddcb9335da87b2bac17","schema_version":"1.0","event_id":"sha256:de29953b6a0698482c6a4dca73877ac6d133c8e0d9be1ddcb9335da87b2bac17"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/bundle.json","state_url":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T13:09:00Z","links":{"resolver":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7","bundle":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/bundle.json","state":"https://pith.science/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/X7FQ3DXEJ6LUXN33MSR2BOS2H7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:X7FQ3DXEJ6LUXN33MSR2BOS2H7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"638c3d96bfecde606324094d27acde41a54160dc7669433b4e5125cf9632302f","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","title_canon_sha256":"34e60341c1ee748dc003c50aaafd089b5762511c95a1268540b74dea7388fa68"},"schema_version":"1.0","source":{"id":"2312.00267","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.00267","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"arxiv_version","alias_value":"2312.00267v3","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.00267","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_12","alias_value":"X7FQ3DXEJ6LU","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_16","alias_value":"X7FQ3DXEJ6LUXN33","created_at":"2026-07-05T10:35:30Z"},{"alias_kind":"pith_short_8","alias_value":"X7FQ3DXE","created_at":"2026-07-05T10:35:30Z"}],"graph_snapshots":[{"event_id":"sha256:de29953b6a0698482c6a4dca73877ac6d133c8e0d9be1ddcb9335da87b2bac17","target":"graph","created_at":"2026-07-05T10:35:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.00267/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Preference-based feedback is important for many applications in machine learning where evaluation of a reward function is not feasible. Notable recent examples arise in preference alignment for large language models, including in reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO). For many applications of preference alignment, the cost of acquiring human feedback can be substantial. In this work, we take advantage of the fact that one can often choose contexts at which to obtain human feedback to most efficiently identify a good policy, and formalize the","authors_text":"Barbara Engelhardt, Ilija Bogunovic, Jeff Schneider, Ojash Neopane, Stefano Ermon, Syrine Belakaria, Vikramjeet Das, Viraj Mehta, Willie Neiswanger, Yijia Dai","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","title":"Sample Efficient Preference Alignment in LLMs via Active Exploration"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.00267","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5b1d2e9df2823ac9ac014a9b3fefc71def35529f0a0b717fb1c6206565869b91","target":"record","created_at":"2026-07-05T10:35:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"638c3d96bfecde606324094d27acde41a54160dc7669433b4e5125cf9632302f","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-01T00:54:02Z","title_canon_sha256":"34e60341c1ee748dc003c50aaafd089b5762511c95a1268540b74dea7388fa68"},"schema_version":"1.0","source":{"id":"2312.00267","kind":"arxiv","version":3}},"canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bfcb0d8ee44f974bb77b64a3a0ba5a3fe255dfeb9db2915fecc54c11494cc187","first_computed_at":"2026-07-05T10:35:30.492069Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:35:30.492069Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KSwCh7nEhlENpKarzXz96O+sYzBlrn5kZDm+5CzLPuDb3sPnI6sD1Wof7DYuj7hw5OEhBC+nR1upD1MJK32yBA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:35:30.492585Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.00267","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5b1d2e9df2823ac9ac014a9b3fefc71def35529f0a0b717fb1c6206565869b91","sha256:de29953b6a0698482c6a4dca73877ac6d133c8e0d9be1ddcb9335da87b2bac17"],"state_sha256":"2544e8d43408acd3bf027b9f8fdf27eb9f11ba7f98988bd6cbae947f89f75c14"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RUfpYbxvd4izHAK7MMcQp3Hw778cpAlIhZtIL4S8lawRT4BU4r3PvkDiubc+26Y51A+HfyIsgyYOi2yK7/ZqDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T13:09:00.517729Z","bundle_sha256":"4f0da1f5fa7476b43367e1ec8022dc0c7d54f5bab58cfca67d9d1be48fd49e21"}}