{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:HGJSQ6ZUZTDFZK7QBRXAUQVE25","short_pith_number":"pith:HGJSQ6ZU","canonical_record":{"source":{"id":"2312.11456","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-18T18:58:42Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"f613085b0bebd5ce64b0b1d0c65a69f658bb9368a6fa5e6eae7dcfa33aecbb09","abstract_canon_sha256":"db8edb732fbccb209037e8a45523088ace1d01ce3cce04e3931e51d4ddae2712"},"schema_version":"1.0"},"canonical_sha256":"3993287b34ccc65cabf00c6e0a42a4d7650757ef5dc8b8fcb2d6d87aede7c47b","source":{"kind":"arxiv","id":"2312.11456","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.11456","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"arxiv_version","alias_value":"2312.11456v4","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.11456","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_12","alias_value":"HGJSQ6ZUZTDF","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_16","alias_value":"HGJSQ6ZUZTDFZK7Q","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_8","alias_value":"HGJSQ6ZU","created_at":"2026-07-05T08:14:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:HGJSQ6ZUZTDFZK7QBRXAUQVE25","target":"record","payload":{"canonical_record":{"source":{"id":"2312.11456","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-18T18:58:42Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"f613085b0bebd5ce64b0b1d0c65a69f658bb9368a6fa5e6eae7dcfa33aecbb09","abstract_canon_sha256":"db8edb732fbccb209037e8a45523088ace1d01ce3cce04e3931e51d4ddae2712"},"schema_version":"1.0"},"canonical_sha256":"3993287b34ccc65cabf00c6e0a42a4d7650757ef5dc8b8fcb2d6d87aede7c47b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:14:02.872490Z","signature_b64":"LyjTcYr9NTZcJtrYRmR1WlwhnHCWxUYlzf9wLsLkwtrYkwGMvOn17t2aki6PqMabYtFH6mXRblPzUKDlcEz4Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3993287b34ccc65cabf00c6e0a42a4d7650757ef5dc8b8fcb2d6d87aede7c47b","last_reissued_at":"2026-07-05T08:14:02.872011Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:14:02.872011Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.11456","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:14:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"c9eSrEikYay/KE6QcuANyeYmvfriSEmOvMsmLYHYlaXFEnybyYDiXjpSFPacpHKsvfNn8MOuI+vmU1xIxBIPCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T17:06:10.553293Z"},"content_sha256":"1be712975edf206b03d412501c6088ba71ed62fbbc9855a6cbbc31ddc4501dea","schema_version":"1.0","event_id":"sha256:1be712975edf206b03d412501c6088ba71ed62fbbc9855a6cbbc31ddc4501dea"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:HGJSQ6ZUZTDFZK7QBRXAUQVE25","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Iterative Preference Learning from Human Feedback: Bridging Theory and Practice for RLHF under KL-Constraint","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Chenlu Ye, HanZe Dong, Han Zhong, Heng Ji, Nan Jiang, Tong Zhang, Wei Xiong, Ziqi Wang","submitted_at":"2023-12-18T18:58:42Z","abstract_excerpt":"This paper studies the alignment process of generative models with Reinforcement Learning from Human Feedback (RLHF). We first identify the primary challenges of existing popular methods like offline PPO and offline DPO as lacking in strategical exploration of the environment. Then, to understand the mathematical principle of RLHF, we consider a standard mathematical formulation, the reverse-KL regularized contextual bandit for RLHF. Despite its widespread practical application, a rigorous theoretical analysis of this formulation remains open. We investigate its behavior in three distinct sett"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.11456","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.11456/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:14:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qyhlpLU7ziiFO4VM7aTib29P97UiSII8mSKK9XpoEsR/uhEpb+KAoYyfMVWw3NDTLBpS0WGN5FedDdYnmk70Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T17:06:10.553677Z"},"content_sha256":"31ff1b965eceb800419776db21b1b57e899267b84dd789da74b9e39a07aac23e","schema_version":"1.0","event_id":"sha256:31ff1b965eceb800419776db21b1b57e899267b84dd789da74b9e39a07aac23e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/bundle.json","state_url":"https://pith.science/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T17:06:10Z","links":{"resolver":"https://pith.science/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25","bundle":"https://pith.science/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/bundle.json","state":"https://pith.science/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HGJSQ6ZUZTDFZK7QBRXAUQVE25/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:HGJSQ6ZUZTDFZK7QBRXAUQVE25","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"db8edb732fbccb209037e8a45523088ace1d01ce3cce04e3931e51d4ddae2712","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-18T18:58:42Z","title_canon_sha256":"f613085b0bebd5ce64b0b1d0c65a69f658bb9368a6fa5e6eae7dcfa33aecbb09"},"schema_version":"1.0","source":{"id":"2312.11456","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.11456","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"arxiv_version","alias_value":"2312.11456v4","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.11456","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_12","alias_value":"HGJSQ6ZUZTDF","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_16","alias_value":"HGJSQ6ZUZTDFZK7Q","created_at":"2026-07-05T08:14:02Z"},{"alias_kind":"pith_short_8","alias_value":"HGJSQ6ZU","created_at":"2026-07-05T08:14:02Z"}],"graph_snapshots":[{"event_id":"sha256:31ff1b965eceb800419776db21b1b57e899267b84dd789da74b9e39a07aac23e","target":"graph","created_at":"2026-07-05T08:14:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.11456/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper studies the alignment process of generative models with Reinforcement Learning from Human Feedback (RLHF). We first identify the primary challenges of existing popular methods like offline PPO and offline DPO as lacking in strategical exploration of the environment. Then, to understand the mathematical principle of RLHF, we consider a standard mathematical formulation, the reverse-KL regularized contextual bandit for RLHF. Despite its widespread practical application, a rigorous theoretical analysis of this formulation remains open. We investigate its behavior in three distinct sett","authors_text":"Chenlu Ye, HanZe Dong, Han Zhong, Heng Ji, Nan Jiang, Tong Zhang, Wei Xiong, Ziqi Wang","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-18T18:58:42Z","title":"Iterative Preference Learning from Human Feedback: Bridging Theory and Practice for RLHF under KL-Constraint"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.11456","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1be712975edf206b03d412501c6088ba71ed62fbbc9855a6cbbc31ddc4501dea","target":"record","created_at":"2026-07-05T08:14:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"db8edb732fbccb209037e8a45523088ace1d01ce3cce04e3931e51d4ddae2712","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-12-18T18:58:42Z","title_canon_sha256":"f613085b0bebd5ce64b0b1d0c65a69f658bb9368a6fa5e6eae7dcfa33aecbb09"},"schema_version":"1.0","source":{"id":"2312.11456","kind":"arxiv","version":4}},"canonical_sha256":"3993287b34ccc65cabf00c6e0a42a4d7650757ef5dc8b8fcb2d6d87aede7c47b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3993287b34ccc65cabf00c6e0a42a4d7650757ef5dc8b8fcb2d6d87aede7c47b","first_computed_at":"2026-07-05T08:14:02.872011Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:14:02.872011Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LyjTcYr9NTZcJtrYRmR1WlwhnHCWxUYlzf9wLsLkwtrYkwGMvOn17t2aki6PqMabYtFH6mXRblPzUKDlcEz4Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T08:14:02.872490Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.11456","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1be712975edf206b03d412501c6088ba71ed62fbbc9855a6cbbc31ddc4501dea","sha256:31ff1b965eceb800419776db21b1b57e899267b84dd789da74b9e39a07aac23e"],"state_sha256":"b5887762f0555492307207861432de380574a4185c4914199b41c5e7dcfba33f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z77UJHRI25SeUy4H492M+CFE1GzWDfJig97HE19x8AW/Dd4W1nMS19+2abWC1c/7hVoQZZ4bfG01BR06fKPCCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T17:06:10.556513Z","bundle_sha256":"a6a2f69e8c4342c4b791f2df437e4119e03fb9843cd97f21fad30f02d7ce6211"}}