{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:T46AK3IHKMBFEZVHSLNMDHWLWI","short_pith_number":"pith:T46AK3IH","canonical_record":{"source":{"id":"2401.16335","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-01-29T17:43:42Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"4f489371328a1af8cc26fffa9e513f196aaf953629844c8f004c2be1d2435eca","abstract_canon_sha256":"bb7bbc795d6445af453cf36ec9cbe2e9f2e8735bb8c319084846119bb65a4fd2"},"schema_version":"1.0"},"canonical_sha256":"9f3c056d0753025266a792dac19ecbb21d883d5fbcf218554aed5cc2c1c2c17c","source":{"kind":"arxiv","id":"2401.16335","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16335","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16335v1","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16335","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_12","alias_value":"T46AK3IHKMBF","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_16","alias_value":"T46AK3IHKMBFEZVH","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_8","alias_value":"T46AK3IH","created_at":"2026-07-05T07:38:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:T46AK3IHKMBFEZVHSLNMDHWLWI","target":"record","payload":{"canonical_record":{"source":{"id":"2401.16335","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-01-29T17:43:42Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"4f489371328a1af8cc26fffa9e513f196aaf953629844c8f004c2be1d2435eca","abstract_canon_sha256":"bb7bbc795d6445af453cf36ec9cbe2e9f2e8735bb8c319084846119bb65a4fd2"},"schema_version":"1.0"},"canonical_sha256":"9f3c056d0753025266a792dac19ecbb21d883d5fbcf218554aed5cc2c1c2c17c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:38:47.626107Z","signature_b64":"vTbp5mKIDGIdRKgHekB72ojlGO+RA32qwdiX4gEqXiYZ9ubeoJPv5F4DDcSgThefmsdrh7m6q8nlkHLklnfXBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9f3c056d0753025266a792dac19ecbb21d883d5fbcf218554aed5cc2c1c2c17c","last_reissued_at":"2026-07-05T07:38:47.625649Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:38:47.625649Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.16335","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QCVUtkTQdqHczasLRkyENDDmZVqjVdmxuza0e0hO/zZB5p2lECrVW9NXqzIeZy0vshhQXCfCZA8dwuInapZGBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T05:03:21.126194Z"},"content_sha256":"89d332184adb18d5866ab1e933908715cffe346ea8ce1b169411a91060c101a9","schema_version":"1.0","event_id":"sha256:89d332184adb18d5866ab1e933908715cffe346ea8ce1b169411a91060c101a9"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:T46AK3IHKMBFEZVHSLNMDHWLWI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Iterative Data Smoothing: Mitigating Reward Overfitting and Overoptimization in RLHF","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Banghua Zhu, Jiantao Jiao, Michael I. Jordan","submitted_at":"2024-01-29T17:43:42Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) is a pivotal technique that aligns language models closely with human-centric values. The initial phase of RLHF involves learning human values using a reward model from ranking data. It is observed that the performance of the reward model degrades after one epoch of training, and optimizing too much against the learned reward model eventually hinders the true objective. This paper delves into these issues, leveraging the theoretical insights to design improved reward learning algorithm termed 'Iterative Data Smoothing' (IDS). The core idea is t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16335","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.16335/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IBGVRl4SMgr1D9g51GbfN/Jav3ihGxpi/n4q4i6jKHX0LywKAm2afHnxUjWczm82PGQIs0hynWYXibRGmhiDAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T05:03:21.126719Z"},"content_sha256":"53392d7985e1d2dd1b2a479ee12aded7387075216b52f35a9e56df65c9f9e407","schema_version":"1.0","event_id":"sha256:53392d7985e1d2dd1b2a479ee12aded7387075216b52f35a9e56df65c9f9e407"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/bundle.json","state_url":"https://pith.science/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T05:03:21Z","links":{"resolver":"https://pith.science/pith/T46AK3IHKMBFEZVHSLNMDHWLWI","bundle":"https://pith.science/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/bundle.json","state":"https://pith.science/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/T46AK3IHKMBFEZVHSLNMDHWLWI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:T46AK3IHKMBFEZVHSLNMDHWLWI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bb7bbc795d6445af453cf36ec9cbe2e9f2e8735bb8c319084846119bb65a4fd2","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-01-29T17:43:42Z","title_canon_sha256":"4f489371328a1af8cc26fffa9e513f196aaf953629844c8f004c2be1d2435eca"},"schema_version":"1.0","source":{"id":"2401.16335","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.16335","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"arxiv_version","alias_value":"2401.16335v1","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.16335","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_12","alias_value":"T46AK3IHKMBF","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_16","alias_value":"T46AK3IHKMBFEZVH","created_at":"2026-07-05T07:38:47Z"},{"alias_kind":"pith_short_8","alias_value":"T46AK3IH","created_at":"2026-07-05T07:38:47Z"}],"graph_snapshots":[{"event_id":"sha256:53392d7985e1d2dd1b2a479ee12aded7387075216b52f35a9e56df65c9f9e407","target":"graph","created_at":"2026-07-05T07:38:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.16335/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) is a pivotal technique that aligns language models closely with human-centric values. The initial phase of RLHF involves learning human values using a reward model from ranking data. It is observed that the performance of the reward model degrades after one epoch of training, and optimizing too much against the learned reward model eventually hinders the true objective. This paper delves into these issues, leveraging the theoretical insights to design improved reward learning algorithm termed 'Iterative Data Smoothing' (IDS). The core idea is t","authors_text":"Banghua Zhu, Jiantao Jiao, Michael I. Jordan","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-01-29T17:43:42Z","title":"Iterative Data Smoothing: Mitigating Reward Overfitting and Overoptimization in RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.16335","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:89d332184adb18d5866ab1e933908715cffe346ea8ce1b169411a91060c101a9","target":"record","created_at":"2026-07-05T07:38:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bb7bbc795d6445af453cf36ec9cbe2e9f2e8735bb8c319084846119bb65a4fd2","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-01-29T17:43:42Z","title_canon_sha256":"4f489371328a1af8cc26fffa9e513f196aaf953629844c8f004c2be1d2435eca"},"schema_version":"1.0","source":{"id":"2401.16335","kind":"arxiv","version":1}},"canonical_sha256":"9f3c056d0753025266a792dac19ecbb21d883d5fbcf218554aed5cc2c1c2c17c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9f3c056d0753025266a792dac19ecbb21d883d5fbcf218554aed5cc2c1c2c17c","first_computed_at":"2026-07-05T07:38:47.625649Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:38:47.625649Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"vTbp5mKIDGIdRKgHekB72ojlGO+RA32qwdiX4gEqXiYZ9ubeoJPv5F4DDcSgThefmsdrh7m6q8nlkHLklnfXBg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:38:47.626107Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.16335","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:89d332184adb18d5866ab1e933908715cffe346ea8ce1b169411a91060c101a9","sha256:53392d7985e1d2dd1b2a479ee12aded7387075216b52f35a9e56df65c9f9e407"],"state_sha256":"3f3477f2cf435374cef7f3e1c1d1eff3ecaa9199d9b79157d058f3d60e677546"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8tKah7yO8FhOX/LnwoAA5wdEslsCK+6OFFANbDvXmJa8VYK0tfqqTZZwvGclwRcZgR/H/PdDd4M1zt5Um0a5Cw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T05:03:21.131967Z","bundle_sha256":"4fa5ee42a8ba3c40d7bb5d3c5d56237310bd6d4caec6459e4a05c5403c8ee23c"}}