{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:CZQI42F53LSHC2G423VBL7RX3W","short_pith_number":"pith:CZQI42F5","canonical_record":{"source":{"id":"2405.19320","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T17:51:42Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"948c5550368b675ca6135d584d7e84ffd7f136b402ac9184efcaf4f5c98f4664","abstract_canon_sha256":"1dd64e9e9843507cc503c6dc9da75a0f9d15f86568c3198ea8c689d43398e940"},"schema_version":"1.0"},"canonical_sha256":"16608e68bddae47168dcd6ea15fe37ddb63cffcdf13a4072ef39b28559a2f8e2","source":{"kind":"arxiv","id":"2405.19320","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19320","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19320v4","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19320","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_12","alias_value":"CZQI42F53LSH","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_16","alias_value":"CZQI42F53LSHC2G4","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_8","alias_value":"CZQI42F5","created_at":"2026-07-05T10:16:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:CZQI42F53LSHC2G423VBL7RX3W","target":"record","payload":{"canonical_record":{"source":{"id":"2405.19320","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T17:51:42Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"948c5550368b675ca6135d584d7e84ffd7f136b402ac9184efcaf4f5c98f4664","abstract_canon_sha256":"1dd64e9e9843507cc503c6dc9da75a0f9d15f86568c3198ea8c689d43398e940"},"schema_version":"1.0"},"canonical_sha256":"16608e68bddae47168dcd6ea15fe37ddb63cffcdf13a4072ef39b28559a2f8e2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:16:34.065951Z","signature_b64":"sdzYIJORKE/AouXn6qad/Zm0sg19Kdv07TBpusGg+k1CBENQBHhOD8nasAanA7UF23mXkYsoj9+BoBxMpCzyCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"16608e68bddae47168dcd6ea15fe37ddb63cffcdf13a4072ef39b28559a2f8e2","last_reissued_at":"2026-07-05T10:16:34.065445Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:16:34.065445Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.19320","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:16:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TRUb400nil5Tm1G9jpe/1JklRi7OOdptreN6rSl8DToJz7//s5RCTe9TIZ6GgdRj9roM8rHoGH+gZR/9+yupDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T13:50:11.220851Z"},"content_sha256":"fe465757231ba5e53dc01ebe2d9835885fc4ba7ae2fed940916caade50840fad","schema_version":"1.0","event_id":"sha256:fe465757231ba5e53dc01ebe2d9835885fc4ba7ae2fed940916caade50840fad"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:CZQI42F53LSHC2G423VBL7RX3W","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Bo Dai, Dale Schuurmans, Hanjun Dai, Jincheng Mei, Katayoon Goshvadi, Sherry Yang, Shicong Cen, Tong Yang, Yuejie Chi","submitted_at":"2024-05-29T17:51:42Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated great promise in aligning large language models (LLMs) with human preference. Depending on the availability of preference data, both online and offline RLHF are active areas of investigation. A key bottleneck is understanding how to incorporate uncertainty estimation in the reward function learned from the preference data for RLHF, regardless of how the preference data is collected. While the principles of optimism or pessimism under uncertainty are well-established in standard reinforcement learning (RL), a practically-impleme"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19320","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.19320/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:16:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0p0ykwRfQDWCNUVgNBESI+CvqhimDS4e/L1itgSVXgJfRUcCUegKswwQ0spOTrZwtO1m5iD/B1SEdoVkkx7nCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T13:50:11.221234Z"},"content_sha256":"eaa55c3f62f7c1ffee4b1b17e77f81ff7637c6c492088ebc251626395795f355","schema_version":"1.0","event_id":"sha256:eaa55c3f62f7c1ffee4b1b17e77f81ff7637c6c492088ebc251626395795f355"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CZQI42F53LSHC2G423VBL7RX3W/bundle.json","state_url":"https://pith.science/pith/CZQI42F53LSHC2G423VBL7RX3W/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CZQI42F53LSHC2G423VBL7RX3W/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T13:50:11Z","links":{"resolver":"https://pith.science/pith/CZQI42F53LSHC2G423VBL7RX3W","bundle":"https://pith.science/pith/CZQI42F53LSHC2G423VBL7RX3W/bundle.json","state":"https://pith.science/pith/CZQI42F53LSHC2G423VBL7RX3W/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CZQI42F53LSHC2G423VBL7RX3W/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:CZQI42F53LSHC2G423VBL7RX3W","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1dd64e9e9843507cc503c6dc9da75a0f9d15f86568c3198ea8c689d43398e940","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T17:51:42Z","title_canon_sha256":"948c5550368b675ca6135d584d7e84ffd7f136b402ac9184efcaf4f5c98f4664"},"schema_version":"1.0","source":{"id":"2405.19320","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19320","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19320v4","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19320","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_12","alias_value":"CZQI42F53LSH","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_16","alias_value":"CZQI42F53LSHC2G4","created_at":"2026-07-05T10:16:34Z"},{"alias_kind":"pith_short_8","alias_value":"CZQI42F5","created_at":"2026-07-05T10:16:34Z"}],"graph_snapshots":[{"event_id":"sha256:eaa55c3f62f7c1ffee4b1b17e77f81ff7637c6c492088ebc251626395795f355","target":"graph","created_at":"2026-07-05T10:16:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.19320/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated great promise in aligning large language models (LLMs) with human preference. Depending on the availability of preference data, both online and offline RLHF are active areas of investigation. A key bottleneck is understanding how to incorporate uncertainty estimation in the reward function learned from the preference data for RLHF, regardless of how the preference data is collected. While the principles of optimism or pessimism under uncertainty are well-established in standard reinforcement learning (RL), a practically-impleme","authors_text":"Bo Dai, Dale Schuurmans, Hanjun Dai, Jincheng Mei, Katayoon Goshvadi, Sherry Yang, Shicong Cen, Tong Yang, Yuejie Chi","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T17:51:42Z","title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19320","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fe465757231ba5e53dc01ebe2d9835885fc4ba7ae2fed940916caade50840fad","target":"record","created_at":"2026-07-05T10:16:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1dd64e9e9843507cc503c6dc9da75a0f9d15f86568c3198ea8c689d43398e940","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T17:51:42Z","title_canon_sha256":"948c5550368b675ca6135d584d7e84ffd7f136b402ac9184efcaf4f5c98f4664"},"schema_version":"1.0","source":{"id":"2405.19320","kind":"arxiv","version":4}},"canonical_sha256":"16608e68bddae47168dcd6ea15fe37ddb63cffcdf13a4072ef39b28559a2f8e2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"16608e68bddae47168dcd6ea15fe37ddb63cffcdf13a4072ef39b28559a2f8e2","first_computed_at":"2026-07-05T10:16:34.065445Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:16:34.065445Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"sdzYIJORKE/AouXn6qad/Zm0sg19Kdv07TBpusGg+k1CBENQBHhOD8nasAanA7UF23mXkYsoj9+BoBxMpCzyCg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:16:34.065951Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.19320","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fe465757231ba5e53dc01ebe2d9835885fc4ba7ae2fed940916caade50840fad","sha256:eaa55c3f62f7c1ffee4b1b17e77f81ff7637c6c492088ebc251626395795f355"],"state_sha256":"405d6b46d86bc0f3cc014109254d2b847abc4dcc14e56877b1899f21480fa6e8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"K4SyI5rUOQ23fut4NIt0a0M+Q69XeIOqPQN0PPe68KCxkAakkEeg/L7YivNHg5ZiLvaXntGzv7U9qbeSmcHlCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T13:50:11.223240Z","bundle_sha256":"16c643c3254fbccf768fe637ee40f3bdb08ae6a29b6ff3301e558f0513de199b"}}