{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KORKIZ3HNYUDFDXBFBVOB7FXFJ","short_pith_number":"pith:KORKIZ3H","canonical_record":{"source":{"id":"2406.09574","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-13T20:25:52Z","cross_cats_sorted":[],"title_canon_sha256":"d939572dbaa2b6ce2d58c3183f10e78b72d75a5b0456c54ad40a2829de540570","abstract_canon_sha256":"0184652cbcfae728a358befb70374742a7a5553dcb05b9986641503cd34fad76"},"schema_version":"1.0"},"canonical_sha256":"53a2a467676e28328ee1286ae0fcb72a4920ae4b4a6dc252982a2a1e3365b569","source":{"kind":"arxiv","id":"2406.09574","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09574","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09574v4","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09574","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_12","alias_value":"KORKIZ3HNYUD","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_16","alias_value":"KORKIZ3HNYUDFDXB","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_8","alias_value":"KORKIZ3H","created_at":"2026-07-05T11:03:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KORKIZ3HNYUDFDXBFBVOB7FXFJ","target":"record","payload":{"canonical_record":{"source":{"id":"2406.09574","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-13T20:25:52Z","cross_cats_sorted":[],"title_canon_sha256":"d939572dbaa2b6ce2d58c3183f10e78b72d75a5b0456c54ad40a2829de540570","abstract_canon_sha256":"0184652cbcfae728a358befb70374742a7a5553dcb05b9986641503cd34fad76"},"schema_version":"1.0"},"canonical_sha256":"53a2a467676e28328ee1286ae0fcb72a4920ae4b4a6dc252982a2a1e3365b569","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:03:50.390862Z","signature_b64":"DqcLwQD922ccP3CugNwRdj7khk/MVBrhRYBhZlsKLbs6CBWClaE62lyS0dyRsmSPejukBEKs1TZyjuaedn5dBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"53a2a467676e28328ee1286ae0fcb72a4920ae4b4a6dc252982a2a1e3365b569","last_reissued_at":"2026-07-05T11:03:50.390450Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:03:50.390450Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.09574","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OGyBcAoelQZhLyBqmRhW8qj7E8DTPy36r623kRuRB2hYcIS195n19pzjUloYQmi4foiWUSDnY6PGQaer11dlCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:53:23.825512Z"},"content_sha256":"b1bd11f9f2c64e100279adc7a391882a097233d75931e58cb4fca623347f4829","schema_version":"1.0","event_id":"sha256:b1bd11f9f2c64e100279adc7a391882a097233d75931e58cb4fca623347f4829"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KORKIZ3HNYUDFDXBFBVOB7FXFJ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Online Bandit Learning with Offline Preference Data for Improved RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Akhil Agnihotri, Deepak Ramachandran, Rahul Jain, Zheng Wen","submitted_at":"2024-06-13T20:25:52Z","abstract_excerpt":"Reinforcement Learning with Human Feedback (RLHF) is at the core of fine-tuning methods for generative AI models for language and images. Such feedback is often sought as rank or preference feedback from human raters, as opposed to eliciting scores since the latter tends to be noisy. On the other hand, RL theory and algorithms predominantly assume that a reward feedback is available. In particular, approaches for online learning that can be helpful in adaptive data collection via active learning cannot incorporate offline preference data. In this paper, we adopt a finite-armed linear bandit mo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09574","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.09574/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Uhc8jSamJE6pXfwAlpWxsiDgi6KCVMvLd6QsmF1yQ6iAQzi/6pclw/dR63IoFZ2ilC7r+GRHE0DWmUp3d/M5Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:53:23.826038Z"},"content_sha256":"d3abb78bd59c215c03a02a6786109b6bb35e69d15c810773e16cc052adea6309","schema_version":"1.0","event_id":"sha256:d3abb78bd59c215c03a02a6786109b6bb35e69d15c810773e16cc052adea6309"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/bundle.json","state_url":"https://pith.science/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T09:53:23Z","links":{"resolver":"https://pith.science/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ","bundle":"https://pith.science/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/bundle.json","state":"https://pith.science/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KORKIZ3HNYUDFDXBFBVOB7FXFJ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KORKIZ3HNYUDFDXBFBVOB7FXFJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0184652cbcfae728a358befb70374742a7a5553dcb05b9986641503cd34fad76","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-13T20:25:52Z","title_canon_sha256":"d939572dbaa2b6ce2d58c3183f10e78b72d75a5b0456c54ad40a2829de540570"},"schema_version":"1.0","source":{"id":"2406.09574","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09574","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09574v4","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09574","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_12","alias_value":"KORKIZ3HNYUD","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_16","alias_value":"KORKIZ3HNYUDFDXB","created_at":"2026-07-05T11:03:50Z"},{"alias_kind":"pith_short_8","alias_value":"KORKIZ3H","created_at":"2026-07-05T11:03:50Z"}],"graph_snapshots":[{"event_id":"sha256:d3abb78bd59c215c03a02a6786109b6bb35e69d15c810773e16cc052adea6309","target":"graph","created_at":"2026-07-05T11:03:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.09574/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning with Human Feedback (RLHF) is at the core of fine-tuning methods for generative AI models for language and images. Such feedback is often sought as rank or preference feedback from human raters, as opposed to eliciting scores since the latter tends to be noisy. On the other hand, RL theory and algorithms predominantly assume that a reward feedback is available. In particular, approaches for online learning that can be helpful in adaptive data collection via active learning cannot incorporate offline preference data. In this paper, we adopt a finite-armed linear bandit mo","authors_text":"Akhil Agnihotri, Deepak Ramachandran, Rahul Jain, Zheng Wen","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-13T20:25:52Z","title":"Online Bandit Learning with Offline Preference Data for Improved RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09574","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b1bd11f9f2c64e100279adc7a391882a097233d75931e58cb4fca623347f4829","target":"record","created_at":"2026-07-05T11:03:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0184652cbcfae728a358befb70374742a7a5553dcb05b9986641503cd34fad76","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-13T20:25:52Z","title_canon_sha256":"d939572dbaa2b6ce2d58c3183f10e78b72d75a5b0456c54ad40a2829de540570"},"schema_version":"1.0","source":{"id":"2406.09574","kind":"arxiv","version":4}},"canonical_sha256":"53a2a467676e28328ee1286ae0fcb72a4920ae4b4a6dc252982a2a1e3365b569","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"53a2a467676e28328ee1286ae0fcb72a4920ae4b4a6dc252982a2a1e3365b569","first_computed_at":"2026-07-05T11:03:50.390450Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:03:50.390450Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"DqcLwQD922ccP3CugNwRdj7khk/MVBrhRYBhZlsKLbs6CBWClaE62lyS0dyRsmSPejukBEKs1TZyjuaedn5dBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:03:50.390862Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.09574","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b1bd11f9f2c64e100279adc7a391882a097233d75931e58cb4fca623347f4829","sha256:d3abb78bd59c215c03a02a6786109b6bb35e69d15c810773e16cc052adea6309"],"state_sha256":"92841142a23469b1597a6c91c47bd40d2b7ef879e466b8fbcc2022ee7d7b388f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rDJ5F6BFkiDblurofmTpti8JQwYWzFzhHf9s1FnNGHOB2oEwoQHwdJ6wR+2a0PzmLHqvQ7azqjt5OlhhB2LnDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T09:53:23.831210Z","bundle_sha256":"71ea06e0eaf8c4ed1c79b4d2bd83075b76accee7b5684a359e707e313b44687f"}}