{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:PLKQWJ5546YAXJUQGBCIIZNYVM","short_pith_number":"pith:PLKQWJ55","canonical_record":{"source":{"id":"2505.07271","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-12T06:48:26Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"999d967108c12adf82ffc57b6c4b3f9f7a8c1327b1029bb07d65d67989c31c6e","abstract_canon_sha256":"abb4211ddf92a00e9aba74c3f611c8c60b7339f20eadfa75423b566910a21631"},"schema_version":"1.0"},"canonical_sha256":"7ad50b27bde7b00ba69030448465b8ab05ede7456ec10bf4b46c20e066ee2b6a","source":{"kind":"arxiv","id":"2505.07271","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.07271","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"arxiv_version","alias_value":"2505.07271v1","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.07271","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_12","alias_value":"PLKQWJ5546YA","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_16","alias_value":"PLKQWJ5546YAXJUQ","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_8","alias_value":"PLKQWJ55","created_at":"2026-07-05T11:01:45Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:PLKQWJ5546YAXJUQGBCIIZNYVM","target":"record","payload":{"canonical_record":{"source":{"id":"2505.07271","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-12T06:48:26Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"999d967108c12adf82ffc57b6c4b3f9f7a8c1327b1029bb07d65d67989c31c6e","abstract_canon_sha256":"abb4211ddf92a00e9aba74c3f611c8c60b7339f20eadfa75423b566910a21631"},"schema_version":"1.0"},"canonical_sha256":"7ad50b27bde7b00ba69030448465b8ab05ede7456ec10bf4b46c20e066ee2b6a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:01:45.262616Z","signature_b64":"A4FPuRIdHXzB65o6tutqhvaoIGRJ8lq1xSSfxs58yY2OsIpi7R68g186NElPk4VCAaTkwtBjTRcCK7ce9RWcAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7ad50b27bde7b00ba69030448465b8ab05ede7456ec10bf4b46c20e066ee2b6a","last_reissued_at":"2026-07-05T11:01:45.262059Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:01:45.262059Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.07271","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:01:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T0UnFvBTwXvkGhDDNxX8EVf7UgbV49K3ezrGgM+IxxENNPjODwDVPf3Jjc0FBXcqvLf1flWNIgmjjfsB0EofCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:04:58.859092Z"},"content_sha256":"9232b89fa0fecf6a93bde9db923a5872fe099c748b81396ac3a021780f3f21e1","schema_version":"1.0","event_id":"sha256:9232b89fa0fecf6a93bde9db923a5872fe099c748b81396ac3a021780f3f21e1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:PLKQWJ5546YAXJUQGBCIIZNYVM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Robustness of Reward Models for Language Model Alignment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Aman Gupta, Eunki Kim, Guijin Son, James Thorne, Jiwoo Hong, Noah Lee, Shao Tang, WooJin Chung","submitted_at":"2025-05-12T06:48:26Z","abstract_excerpt":"The Bradley-Terry (BT) model is widely practiced in reward modeling for reinforcement learning with human feedback (RLHF). Despite its effectiveness, reward models (RMs) trained with BT model loss are prone to over-optimization, losing generalizability to unseen input distributions. In this paper, we study the cause of over-optimization in RM training and its downstream effects on the RLHF procedure, accentuating the importance of distributional robustness of RMs in unseen data. First, we show that the excessive dispersion of hidden state norms is the main source of over-optimization. Then, we"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.07271","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.07271/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:01:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"toVgxega9tVmgCiqm2ciLxP6zT45KLmPfoVx65MuqSUSQsqiFCgcqK3e0kP/0w22UhSm9Qtz3dzCjiSKCwAQBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:04:58.859480Z"},"content_sha256":"8d4764ba95a32ed43548e165a1054282abad1ef8147f22e84a3912fcf083b7c2","schema_version":"1.0","event_id":"sha256:8d4764ba95a32ed43548e165a1054282abad1ef8147f22e84a3912fcf083b7c2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/bundle.json","state_url":"https://pith.science/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T19:04:58Z","links":{"resolver":"https://pith.science/pith/PLKQWJ5546YAXJUQGBCIIZNYVM","bundle":"https://pith.science/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/bundle.json","state":"https://pith.science/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PLKQWJ5546YAXJUQGBCIIZNYVM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:PLKQWJ5546YAXJUQGBCIIZNYVM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"abb4211ddf92a00e9aba74c3f611c8c60b7339f20eadfa75423b566910a21631","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-12T06:48:26Z","title_canon_sha256":"999d967108c12adf82ffc57b6c4b3f9f7a8c1327b1029bb07d65d67989c31c6e"},"schema_version":"1.0","source":{"id":"2505.07271","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.07271","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"arxiv_version","alias_value":"2505.07271v1","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.07271","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_12","alias_value":"PLKQWJ5546YA","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_16","alias_value":"PLKQWJ5546YAXJUQ","created_at":"2026-07-05T11:01:45Z"},{"alias_kind":"pith_short_8","alias_value":"PLKQWJ55","created_at":"2026-07-05T11:01:45Z"}],"graph_snapshots":[{"event_id":"sha256:8d4764ba95a32ed43548e165a1054282abad1ef8147f22e84a3912fcf083b7c2","target":"graph","created_at":"2026-07-05T11:01:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.07271/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The Bradley-Terry (BT) model is widely practiced in reward modeling for reinforcement learning with human feedback (RLHF). Despite its effectiveness, reward models (RMs) trained with BT model loss are prone to over-optimization, losing generalizability to unseen input distributions. In this paper, we study the cause of over-optimization in RM training and its downstream effects on the RLHF procedure, accentuating the importance of distributional robustness of RMs in unseen data. First, we show that the excessive dispersion of hidden state norms is the main source of over-optimization. Then, we","authors_text":"Aman Gupta, Eunki Kim, Guijin Son, James Thorne, Jiwoo Hong, Noah Lee, Shao Tang, WooJin Chung","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-12T06:48:26Z","title":"On the Robustness of Reward Models for Language Model Alignment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.07271","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9232b89fa0fecf6a93bde9db923a5872fe099c748b81396ac3a021780f3f21e1","target":"record","created_at":"2026-07-05T11:01:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"abb4211ddf92a00e9aba74c3f611c8c60b7339f20eadfa75423b566910a21631","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-12T06:48:26Z","title_canon_sha256":"999d967108c12adf82ffc57b6c4b3f9f7a8c1327b1029bb07d65d67989c31c6e"},"schema_version":"1.0","source":{"id":"2505.07271","kind":"arxiv","version":1}},"canonical_sha256":"7ad50b27bde7b00ba69030448465b8ab05ede7456ec10bf4b46c20e066ee2b6a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7ad50b27bde7b00ba69030448465b8ab05ede7456ec10bf4b46c20e066ee2b6a","first_computed_at":"2026-07-05T11:01:45.262059Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:01:45.262059Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A4FPuRIdHXzB65o6tutqhvaoIGRJ8lq1xSSfxs58yY2OsIpi7R68g186NElPk4VCAaTkwtBjTRcCK7ce9RWcAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:01:45.262616Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.07271","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9232b89fa0fecf6a93bde9db923a5872fe099c748b81396ac3a021780f3f21e1","sha256:8d4764ba95a32ed43548e165a1054282abad1ef8147f22e84a3912fcf083b7c2"],"state_sha256":"bf1f859a574cc6ed26e2b5eb15988c9fe6ccc5d10f0c8e624274f65a20231fc5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lwmtK2NQ5+Uo1fCihp1MFHFjH3tqvh0dsWCg7donCqcGncLwY30dJxBKOgx2XJDpaEP1b5ewq8jGJark1jHIDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T19:04:58.861464Z","bundle_sha256":"7d9fdd81739b100de55af0c0e81367720800b66cfebcde98431f171701e327f2"}}