{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YXEY3DTOWRB4JSGMBTGFRIQATX","short_pith_number":"pith:YXEY3DTO","canonical_record":{"source":{"id":"2402.07319","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-11T22:40:12Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"9bd25a5fdecb6033539a7c5e86ac23cd1a7e685cf72a8c7ea7549b3cc4f105ae","abstract_canon_sha256":"f8743a06e8a418f9701bf8db641cbbc67c6b9b5c5f733bf2c003b2ab676c2d2a"},"schema_version":"1.0"},"canonical_sha256":"c5c98d8e6eb443c4c8cc0ccc58a2009dd9213067635aeaaed30efcb7595efb66","source":{"kind":"arxiv","id":"2402.07319","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.07319","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"arxiv_version","alias_value":"2402.07319v1","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.07319","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_12","alias_value":"YXEY3DTOWRB4","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_16","alias_value":"YXEY3DTOWRB4JSGM","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_8","alias_value":"YXEY3DTO","created_at":"2026-07-05T07:44:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YXEY3DTOWRB4JSGMBTGFRIQATX","target":"record","payload":{"canonical_record":{"source":{"id":"2402.07319","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-11T22:40:12Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"9bd25a5fdecb6033539a7c5e86ac23cd1a7e685cf72a8c7ea7549b3cc4f105ae","abstract_canon_sha256":"f8743a06e8a418f9701bf8db641cbbc67c6b9b5c5f733bf2c003b2ab676c2d2a"},"schema_version":"1.0"},"canonical_sha256":"c5c98d8e6eb443c4c8cc0ccc58a2009dd9213067635aeaaed30efcb7595efb66","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:44:04.509588Z","signature_b64":"nirPK5eThX54RfoJXZJZ/5NqYos9W1+kQoM+UARJL33dwk3cdlKJMRJnEXXPlRPIYwwBNNgu7uPT3GjF/4yBCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c5c98d8e6eb443c4c8cc0ccc58a2009dd9213067635aeaaed30efcb7595efb66","last_reissued_at":"2026-07-05T07:44:04.509149Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:44:04.509149Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.07319","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:44:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sAWWKu3BfKyCBn2w9QZfkLrpot2ouatFJSSBID4LYY0zcHqeEgIAEfLtCEIVbc6cGOnCiEeG2iR/OLByRTC4AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:26:53.877833Z"},"content_sha256":"5c7e0fe90e7ab5c7e68da901cffc64ac36892b915316685954b7ab797c51c762","schema_version":"1.0","event_id":"sha256:5c7e0fe90e7ab5c7e68da901cffc64ac36892b915316685954b7ab797c51c762"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YXEY3DTOWRB4JSGMBTGFRIQATX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ODIN: Disentangled Reward Mitigates Hacking in RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Bryan Catanzaro, Chen Zhu, Davit Soselia, Heng Huang, Jiuhai Chen, Lichang Chen, Mohammad Shoeybi, Tianyi Zhou, Tom Goldstein","submitted_at":"2024-02-11T22:40:12Z","abstract_excerpt":"In this work, we study the issue of reward hacking on the response length, a challenge emerging in Reinforcement Learning from Human Feedback (RLHF) on LLMs. A well-formatted, verbose but less helpful response from the LLMs can often deceive LLMs or even human evaluators to achieve high scores. The same issue also holds for some reward models in RL. To address the challenges in both training and evaluation, we establish a more reliable evaluation protocol for comparing different training configurations, which inspects the trade-off between LLM evaluation score and response length obtained by v"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.07319","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.07319/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:44:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"geKlju+dxOUv8SCWtPXhPd3CMzAdaSOp/sjO6XqBek53pnTjqI+Lk6LrBocN3S1WosJ8aLSa9QlbGT7PK6M8AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:26:53.878369Z"},"content_sha256":"2be49424de0d0958983facf5a9100366779dca5d9544d113381de678ff488aca","schema_version":"1.0","event_id":"sha256:2be49424de0d0958983facf5a9100366779dca5d9544d113381de678ff488aca"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/bundle.json","state_url":"https://pith.science/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:26:53Z","links":{"resolver":"https://pith.science/pith/YXEY3DTOWRB4JSGMBTGFRIQATX","bundle":"https://pith.science/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/bundle.json","state":"https://pith.science/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YXEY3DTOWRB4JSGMBTGFRIQATX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YXEY3DTOWRB4JSGMBTGFRIQATX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f8743a06e8a418f9701bf8db641cbbc67c6b9b5c5f733bf2c003b2ab676c2d2a","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-11T22:40:12Z","title_canon_sha256":"9bd25a5fdecb6033539a7c5e86ac23cd1a7e685cf72a8c7ea7549b3cc4f105ae"},"schema_version":"1.0","source":{"id":"2402.07319","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.07319","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"arxiv_version","alias_value":"2402.07319v1","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.07319","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_12","alias_value":"YXEY3DTOWRB4","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_16","alias_value":"YXEY3DTOWRB4JSGM","created_at":"2026-07-05T07:44:04Z"},{"alias_kind":"pith_short_8","alias_value":"YXEY3DTO","created_at":"2026-07-05T07:44:04Z"}],"graph_snapshots":[{"event_id":"sha256:2be49424de0d0958983facf5a9100366779dca5d9544d113381de678ff488aca","target":"graph","created_at":"2026-07-05T07:44:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.07319/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this work, we study the issue of reward hacking on the response length, a challenge emerging in Reinforcement Learning from Human Feedback (RLHF) on LLMs. A well-formatted, verbose but less helpful response from the LLMs can often deceive LLMs or even human evaluators to achieve high scores. The same issue also holds for some reward models in RL. To address the challenges in both training and evaluation, we establish a more reliable evaluation protocol for comparing different training configurations, which inspects the trade-off between LLM evaluation score and response length obtained by v","authors_text":"Bryan Catanzaro, Chen Zhu, Davit Soselia, Heng Huang, Jiuhai Chen, Lichang Chen, Mohammad Shoeybi, Tianyi Zhou, Tom Goldstein","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-11T22:40:12Z","title":"ODIN: Disentangled Reward Mitigates Hacking in RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.07319","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5c7e0fe90e7ab5c7e68da901cffc64ac36892b915316685954b7ab797c51c762","target":"record","created_at":"2026-07-05T07:44:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f8743a06e8a418f9701bf8db641cbbc67c6b9b5c5f733bf2c003b2ab676c2d2a","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-11T22:40:12Z","title_canon_sha256":"9bd25a5fdecb6033539a7c5e86ac23cd1a7e685cf72a8c7ea7549b3cc4f105ae"},"schema_version":"1.0","source":{"id":"2402.07319","kind":"arxiv","version":1}},"canonical_sha256":"c5c98d8e6eb443c4c8cc0ccc58a2009dd9213067635aeaaed30efcb7595efb66","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c5c98d8e6eb443c4c8cc0ccc58a2009dd9213067635aeaaed30efcb7595efb66","first_computed_at":"2026-07-05T07:44:04.509149Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:44:04.509149Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nirPK5eThX54RfoJXZJZ/5NqYos9W1+kQoM+UARJL33dwk3cdlKJMRJnEXXPlRPIYwwBNNgu7uPT3GjF/4yBCg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:44:04.509588Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.07319","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5c7e0fe90e7ab5c7e68da901cffc64ac36892b915316685954b7ab797c51c762","sha256:2be49424de0d0958983facf5a9100366779dca5d9544d113381de678ff488aca"],"state_sha256":"c61456641636ba74ca35c5f160c68922c05ad43272282037c38492fc243e2fd2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lyCsYRLkcmOdqzWmsE8uvcBAVx7hLhtPTZtKIkVRAwm5kgPxCOdr7XdtsN2JqvCTWUgDV9lQgMiUDBiDiiazCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:26:53.883304Z","bundle_sha256":"5d373d9a8cc0ecc67bce0f90a93b645152a96bb88e11cc213d8344853027a801"}}