{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:WUYXD5C6FUMECSFO6SNKXR33PQ","short_pith_number":"pith:WUYXD5C6","canonical_record":{"source":{"id":"2402.09345","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-14T17:49:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"70dddbf5fc70bf24f3133c6d64a579cba9dfc35d62a5b3c6893c5151a1b50b51","abstract_canon_sha256":"54470a24f03b6f43ca80dc6fe9aa7523869e8649dd6f6e2316fc3689ab554bf6"},"schema_version":"1.0"},"canonical_sha256":"b53171f45e2d184148aef49aabc77b7c258dfaea4bf329f5d979cd6a5f8f80f5","source":{"kind":"arxiv","id":"2402.09345","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.09345","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"arxiv_version","alias_value":"2402.09345v5","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.09345","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_12","alias_value":"WUYXD5C6FUME","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_16","alias_value":"WUYXD5C6FUMECSFO","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_8","alias_value":"WUYXD5C6","created_at":"2026-07-05T09:29:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:WUYXD5C6FUMECSFO6SNKXR33PQ","target":"record","payload":{"canonical_record":{"source":{"id":"2402.09345","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-14T17:49:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"70dddbf5fc70bf24f3133c6d64a579cba9dfc35d62a5b3c6893c5151a1b50b51","abstract_canon_sha256":"54470a24f03b6f43ca80dc6fe9aa7523869e8649dd6f6e2316fc3689ab554bf6"},"schema_version":"1.0"},"canonical_sha256":"b53171f45e2d184148aef49aabc77b7c258dfaea4bf329f5d979cd6a5f8f80f5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:29:26.252649Z","signature_b64":"NXGvpBRcRRaoJhWjUG5KhTRewXa3NbowSZOjyA4xi1m9Bva4owgz2FFFSVUobafxS3r3XD4FPzqrfw1zhrqdBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b53171f45e2d184148aef49aabc77b7c258dfaea4bf329f5d979cd6a5f8f80f5","last_reissued_at":"2026-07-05T09:29:26.252082Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:29:26.252082Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.09345","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lnGElFQzv5PWf3JSE9jDbdn/draAGP8SqiVD9LWWYT+kbfXuYPvSK4ZI2QbWHa/nr2LHEceznFN6rA0aIR4EDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T19:48:03.455409Z"},"content_sha256":"53a54329d5c6440c905eaf61bc2101e26fcba9694282b278cfab40edd6911f12","schema_version":"1.0","event_id":"sha256:53a54329d5c6440c905eaf61bc2101e26fcba9694282b278cfab40edd6911f12"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:WUYXD5C6FUMECSFO6SNKXR33PQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"InfoRM: Mitigating Reward Hacking in RLHF via Information-Theoretic Reward Modeling","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Dacheng Tao, Lefei Zhang, Liang Ding, Rong Bao, Sen Zhang, Yuchun Miao","submitted_at":"2024-02-14T17:49:07Z","abstract_excerpt":"Despite the success of reinforcement learning from human feedback (RLHF) in aligning language models with human values, reward hacking, also termed reward overoptimization, remains a critical challenge. This issue primarily arises from reward misgeneralization, where reward models (RMs) compute reward using spurious features that are irrelevant to human preferences. In this work, we tackle this problem from an information-theoretic perspective and propose a framework for reward modeling, namely InfoRM, by introducing a variational information bottleneck objective to filter out irrelevant infor"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.09345","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.09345/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EZKUCWoNKlDtshA2wEEIVJiXwxtuORB81IXHEUSaPt3GwoHHC6mEDyO73f/kgVjI8OjE4YcsgPYvCmWEPS0ABg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T19:48:03.455956Z"},"content_sha256":"1ab0d9be59174cadcc2d9f175bb19da0752b4d4155bceebb683d692db5077464","schema_version":"1.0","event_id":"sha256:1ab0d9be59174cadcc2d9f175bb19da0752b4d4155bceebb683d692db5077464"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/bundle.json","state_url":"https://pith.science/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T19:48:03Z","links":{"resolver":"https://pith.science/pith/WUYXD5C6FUMECSFO6SNKXR33PQ","bundle":"https://pith.science/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/bundle.json","state":"https://pith.science/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WUYXD5C6FUMECSFO6SNKXR33PQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WUYXD5C6FUMECSFO6SNKXR33PQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"54470a24f03b6f43ca80dc6fe9aa7523869e8649dd6f6e2316fc3689ab554bf6","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-14T17:49:07Z","title_canon_sha256":"70dddbf5fc70bf24f3133c6d64a579cba9dfc35d62a5b3c6893c5151a1b50b51"},"schema_version":"1.0","source":{"id":"2402.09345","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.09345","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"arxiv_version","alias_value":"2402.09345v5","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.09345","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_12","alias_value":"WUYXD5C6FUME","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_16","alias_value":"WUYXD5C6FUMECSFO","created_at":"2026-07-05T09:29:26Z"},{"alias_kind":"pith_short_8","alias_value":"WUYXD5C6","created_at":"2026-07-05T09:29:26Z"}],"graph_snapshots":[{"event_id":"sha256:1ab0d9be59174cadcc2d9f175bb19da0752b4d4155bceebb683d692db5077464","target":"graph","created_at":"2026-07-05T09:29:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.09345/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Despite the success of reinforcement learning from human feedback (RLHF) in aligning language models with human values, reward hacking, also termed reward overoptimization, remains a critical challenge. This issue primarily arises from reward misgeneralization, where reward models (RMs) compute reward using spurious features that are irrelevant to human preferences. In this work, we tackle this problem from an information-theoretic perspective and propose a framework for reward modeling, namely InfoRM, by introducing a variational information bottleneck objective to filter out irrelevant infor","authors_text":"Dacheng Tao, Lefei Zhang, Liang Ding, Rong Bao, Sen Zhang, Yuchun Miao","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-14T17:49:07Z","title":"InfoRM: Mitigating Reward Hacking in RLHF via Information-Theoretic Reward Modeling"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.09345","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53a54329d5c6440c905eaf61bc2101e26fcba9694282b278cfab40edd6911f12","target":"record","created_at":"2026-07-05T09:29:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"54470a24f03b6f43ca80dc6fe9aa7523869e8649dd6f6e2316fc3689ab554bf6","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-14T17:49:07Z","title_canon_sha256":"70dddbf5fc70bf24f3133c6d64a579cba9dfc35d62a5b3c6893c5151a1b50b51"},"schema_version":"1.0","source":{"id":"2402.09345","kind":"arxiv","version":5}},"canonical_sha256":"b53171f45e2d184148aef49aabc77b7c258dfaea4bf329f5d979cd6a5f8f80f5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b53171f45e2d184148aef49aabc77b7c258dfaea4bf329f5d979cd6a5f8f80f5","first_computed_at":"2026-07-05T09:29:26.252082Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:29:26.252082Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NXGvpBRcRRaoJhWjUG5KhTRewXa3NbowSZOjyA4xi1m9Bva4owgz2FFFSVUobafxS3r3XD4FPzqrfw1zhrqdBg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:29:26.252649Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.09345","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53a54329d5c6440c905eaf61bc2101e26fcba9694282b278cfab40edd6911f12","sha256:1ab0d9be59174cadcc2d9f175bb19da0752b4d4155bceebb683d692db5077464"],"state_sha256":"f4551c325290df40cca484d3d488e330a7f163bfcb6f56fd7d8aa01f948b7714"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0WLQRN5hgaGPM0iZ12SZD4Umau8Ks/MzUJ0ih8Q1vLRHE73Bfi4Wb5KgB9II+UuyDLhdqr/QjbHuhJ8ZewHMDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T19:48:03.461788Z","bundle_sha256":"a2c1a4dfa6aee70d54f6b24029c7893b01191d80730ceec959d402e4249fb9e6"}}