{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KGT4TV2MOU5MYNO2IFHIJ5GNDY","short_pith_number":"pith:KGT4TV2M","canonical_record":{"source":{"id":"2406.07971","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-12T07:52:17Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5d20f325d0e0b547193060bf07387a9087e35c024f29664aca5284251d9c988c","abstract_canon_sha256":"d4cb22334834e27fb357523f06f08b7198aded77609991767fddf5c32c8510d2"},"schema_version":"1.0"},"canonical_sha256":"51a7c9d74c753acc35da414e84f4cd1e1f10c3624e10a46d2fd133727d1a633a","source":{"kind":"arxiv","id":"2406.07971","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.07971","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"arxiv_version","alias_value":"2406.07971v2","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.07971","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_12","alias_value":"KGT4TV2MOU5M","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_16","alias_value":"KGT4TV2MOU5MYNO2","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_8","alias_value":"KGT4TV2M","created_at":"2026-07-05T08:31:17Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KGT4TV2MOU5MYNO2IFHIJ5GNDY","target":"record","payload":{"canonical_record":{"source":{"id":"2406.07971","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-12T07:52:17Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5d20f325d0e0b547193060bf07387a9087e35c024f29664aca5284251d9c988c","abstract_canon_sha256":"d4cb22334834e27fb357523f06f08b7198aded77609991767fddf5c32c8510d2"},"schema_version":"1.0"},"canonical_sha256":"51a7c9d74c753acc35da414e84f4cd1e1f10c3624e10a46d2fd133727d1a633a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:31:17.318259Z","signature_b64":"mlit0DgQKEeDgMG9RA49Csw6dgTbJWUpvB0WC/JRI9xlHZBnHP1z/o1pXff2AUx/IThB2GWm23DGuux3RNcBBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"51a7c9d74c753acc35da414e84f4cd1e1f10c3624e10a46d2fd133727d1a633a","last_reissued_at":"2026-07-05T08:31:17.317639Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:31:17.317639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.07971","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:31:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FoWZCvgAdqt2iIqiYlJUC9ZGDVa/wqqVg2HcfJDw0PIbP6RzV3FdjEUGt2lPCrHrV4F0vegwcTGXesqZWjxsAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:54:15.780507Z"},"content_sha256":"306e4bd1a6ee1fc1174ac579c9785245acb426cfebeca145841fcc8200847d86","schema_version":"1.0","event_id":"sha256:306e4bd1a6ee1fc1174ac579c9785245acb426cfebeca145841fcc8200847d86"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KGT4TV2MOU5MYNO2IFHIJ5GNDY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"It Takes Two: On the Seamlessness between Reward and Policy Model in RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Beidi Chen, Huaxiu Yao, Lingfeng Shen, Taiming Lu, Weiting Tan, Xinyu Yang","submitted_at":"2024-06-12T07:52:17Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) involves training policy models (PMs) and reward models (RMs) to align language models with human preferences. Instead of focusing solely on PMs and RMs independently, we propose to examine their interactions during fine-tuning, introducing the concept of seamlessness. Our study starts with observing the saturation phenomenon, where continual improvements in RM and PM do not translate into RLHF progress. Our analysis shows that RMs fail to assign proper scores to PM responses, resulting in a 35% mismatch rate with human preferences, highlightin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.07971","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.07971/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:31:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L6SzWJjtjlLvAzV1Q2Rz0l7gQ+87EJUWxkPsFD0ZIft97w0VjpT4H6+AhuNRFcJWOQhb2qVTf7PYCRcVDB7kDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:54:15.780884Z"},"content_sha256":"a0ea6f4e0537e78e794b0a01f8e48ccf09724e78f901a43a508567de18360b06","schema_version":"1.0","event_id":"sha256:a0ea6f4e0537e78e794b0a01f8e48ccf09724e78f901a43a508567de18360b06"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/bundle.json","state_url":"https://pith.science/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T05:54:15Z","links":{"resolver":"https://pith.science/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY","bundle":"https://pith.science/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/bundle.json","state":"https://pith.science/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KGT4TV2MOU5MYNO2IFHIJ5GNDY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KGT4TV2MOU5MYNO2IFHIJ5GNDY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d4cb22334834e27fb357523f06f08b7198aded77609991767fddf5c32c8510d2","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-12T07:52:17Z","title_canon_sha256":"5d20f325d0e0b547193060bf07387a9087e35c024f29664aca5284251d9c988c"},"schema_version":"1.0","source":{"id":"2406.07971","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.07971","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"arxiv_version","alias_value":"2406.07971v2","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.07971","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_12","alias_value":"KGT4TV2MOU5M","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_16","alias_value":"KGT4TV2MOU5MYNO2","created_at":"2026-07-05T08:31:17Z"},{"alias_kind":"pith_short_8","alias_value":"KGT4TV2M","created_at":"2026-07-05T08:31:17Z"}],"graph_snapshots":[{"event_id":"sha256:a0ea6f4e0537e78e794b0a01f8e48ccf09724e78f901a43a508567de18360b06","target":"graph","created_at":"2026-07-05T08:31:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.07971/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) involves training policy models (PMs) and reward models (RMs) to align language models with human preferences. Instead of focusing solely on PMs and RMs independently, we propose to examine their interactions during fine-tuning, introducing the concept of seamlessness. Our study starts with observing the saturation phenomenon, where continual improvements in RM and PM do not translate into RLHF progress. Our analysis shows that RMs fail to assign proper scores to PM responses, resulting in a 35% mismatch rate with human preferences, highlightin","authors_text":"Beidi Chen, Huaxiu Yao, Lingfeng Shen, Taiming Lu, Weiting Tan, Xinyu Yang","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-12T07:52:17Z","title":"It Takes Two: On the Seamlessness between Reward and Policy Model in RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.07971","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:306e4bd1a6ee1fc1174ac579c9785245acb426cfebeca145841fcc8200847d86","target":"record","created_at":"2026-07-05T08:31:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d4cb22334834e27fb357523f06f08b7198aded77609991767fddf5c32c8510d2","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-12T07:52:17Z","title_canon_sha256":"5d20f325d0e0b547193060bf07387a9087e35c024f29664aca5284251d9c988c"},"schema_version":"1.0","source":{"id":"2406.07971","kind":"arxiv","version":2}},"canonical_sha256":"51a7c9d74c753acc35da414e84f4cd1e1f10c3624e10a46d2fd133727d1a633a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"51a7c9d74c753acc35da414e84f4cd1e1f10c3624e10a46d2fd133727d1a633a","first_computed_at":"2026-07-05T08:31:17.317639Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:31:17.317639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mlit0DgQKEeDgMG9RA49Csw6dgTbJWUpvB0WC/JRI9xlHZBnHP1z/o1pXff2AUx/IThB2GWm23DGuux3RNcBBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:31:17.318259Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.07971","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:306e4bd1a6ee1fc1174ac579c9785245acb426cfebeca145841fcc8200847d86","sha256:a0ea6f4e0537e78e794b0a01f8e48ccf09724e78f901a43a508567de18360b06"],"state_sha256":"dc046ea00e82049adce2ae175fb143e9ed4446cf303c55f335dce209d124e41f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UwYu+/11X4PgBLZnLZwE9QVMEvIQtr7i6i/a0OgsDgTuMAz1m2F+4WUPZtSPjTw1ttM3WzInTv2lIjfhxcr1CA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T05:54:15.783024Z","bundle_sha256":"bc623fd20b55d2e863e8f114d179679fda19f4bc0b651ca6f8039f28d0e13060"}}