{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:TORMKZ2LPG547MHMWUUBEDTLXI","short_pith_number":"pith:TORMKZ2L","canonical_record":{"source":{"id":"2507.15507","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-21T11:19:04Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"0b1f7e0f3e30a40abec2e0e96f4a4bbc030561a8c0f6baf8a1f8a968b189a353","abstract_canon_sha256":"989339f54f062810c13e13c98d33a4441d2f4b5a45b37bfe77a46d21be4cc9ed"},"schema_version":"1.0"},"canonical_sha256":"9ba2c5674b79bbcfb0ecb528120e6bba3346251ba6451e1be15f8d329b734929","source":{"kind":"arxiv","id":"2507.15507","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.15507","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"arxiv_version","alias_value":"2507.15507v1","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.15507","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_12","alias_value":"TORMKZ2LPG54","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_16","alias_value":"TORMKZ2LPG547MHM","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_8","alias_value":"TORMKZ2L","created_at":"2026-07-05T11:40:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:TORMKZ2LPG547MHMWUUBEDTLXI","target":"record","payload":{"canonical_record":{"source":{"id":"2507.15507","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-21T11:19:04Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"0b1f7e0f3e30a40abec2e0e96f4a4bbc030561a8c0f6baf8a1f8a968b189a353","abstract_canon_sha256":"989339f54f062810c13e13c98d33a4441d2f4b5a45b37bfe77a46d21be4cc9ed"},"schema_version":"1.0"},"canonical_sha256":"9ba2c5674b79bbcfb0ecb528120e6bba3346251ba6451e1be15f8d329b734929","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:40:38.322087Z","signature_b64":"vCK+HHRZusXo0S4D++iqK5wGstKzcSqsXCpR+k86T+HyabZ3ZkGl/VVeC2fy5N6+FbJFfKBhQajGa9FK16RfDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9ba2c5674b79bbcfb0ecb528120e6bba3346251ba6451e1be15f8d329b734929","last_reissued_at":"2026-07-05T11:40:38.321694Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:40:38.321694Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.15507","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:40:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mguaVp5XXObffWA3baeoyZUo1hcj9uJlYhqlr63znhutG//SB5JdkWGzYtnrgy5Pph9I3bfPhQaC4ecqQ12dCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:08:32.401702Z"},"content_sha256":"b4ff5214c83fbaa9378b1874f80271e43764fed156de46c9cba690f3d3dce30d","schema_version":"1.0","event_id":"sha256:b4ff5214c83fbaa9378b1874f80271e43764fed156de46c9cba690f3d3dce30d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:TORMKZ2LPG547MHMWUUBEDTLXI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Off-Policy Corrected Reward Modeling for Reinforcement Learning from Human Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Johannes Ackermann, Masashi Sugiyama, Takashi Ishida","submitted_at":"2025-07-21T11:19:04Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) allows us to train models, such as language models (LMs), to follow complex human preferences. In RLHF for LMs, we first train an LM using supervised fine-tuning, sample pairs of responses, obtain human feedback, and use the resulting data to train a reward model (RM). RL methods are then used to train the LM to maximize the reward given by the RM. As training progresses, the responses generated by the LM no longer resemble the responses seen by the RM during training, leading to the RM becoming inaccurate. The score given by the RM keeps incre"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.15507","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.15507/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:40:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OW0lAE1Ub/Rg8CDz4gF7YbjEzAZcMjsFD1kwgQpSkDtBu6PDJzjtCWE05omU9De8Hx1BFCVhr5Q1LMB+5dLcDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:08:32.402072Z"},"content_sha256":"7fe6ffb6399778563ef1654c611b434e62cd4cc57f09d944af7c7826ac0cdee7","schema_version":"1.0","event_id":"sha256:7fe6ffb6399778563ef1654c611b434e62cd4cc57f09d944af7c7826ac0cdee7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TORMKZ2LPG547MHMWUUBEDTLXI/bundle.json","state_url":"https://pith.science/pith/TORMKZ2LPG547MHMWUUBEDTLXI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TORMKZ2LPG547MHMWUUBEDTLXI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T19:08:32Z","links":{"resolver":"https://pith.science/pith/TORMKZ2LPG547MHMWUUBEDTLXI","bundle":"https://pith.science/pith/TORMKZ2LPG547MHMWUUBEDTLXI/bundle.json","state":"https://pith.science/pith/TORMKZ2LPG547MHMWUUBEDTLXI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TORMKZ2LPG547MHMWUUBEDTLXI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:TORMKZ2LPG547MHMWUUBEDTLXI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"989339f54f062810c13e13c98d33a4441d2f4b5a45b37bfe77a46d21be4cc9ed","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-21T11:19:04Z","title_canon_sha256":"0b1f7e0f3e30a40abec2e0e96f4a4bbc030561a8c0f6baf8a1f8a968b189a353"},"schema_version":"1.0","source":{"id":"2507.15507","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.15507","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"arxiv_version","alias_value":"2507.15507v1","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.15507","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_12","alias_value":"TORMKZ2LPG54","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_16","alias_value":"TORMKZ2LPG547MHM","created_at":"2026-07-05T11:40:38Z"},{"alias_kind":"pith_short_8","alias_value":"TORMKZ2L","created_at":"2026-07-05T11:40:38Z"}],"graph_snapshots":[{"event_id":"sha256:7fe6ffb6399778563ef1654c611b434e62cd4cc57f09d944af7c7826ac0cdee7","target":"graph","created_at":"2026-07-05T11:40:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.15507/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) allows us to train models, such as language models (LMs), to follow complex human preferences. In RLHF for LMs, we first train an LM using supervised fine-tuning, sample pairs of responses, obtain human feedback, and use the resulting data to train a reward model (RM). RL methods are then used to train the LM to maximize the reward given by the RM. As training progresses, the responses generated by the LM no longer resemble the responses seen by the RM during training, leading to the RM becoming inaccurate. The score given by the RM keeps incre","authors_text":"Johannes Ackermann, Masashi Sugiyama, Takashi Ishida","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-21T11:19:04Z","title":"Off-Policy Corrected Reward Modeling for Reinforcement Learning from Human Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.15507","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b4ff5214c83fbaa9378b1874f80271e43764fed156de46c9cba690f3d3dce30d","target":"record","created_at":"2026-07-05T11:40:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"989339f54f062810c13e13c98d33a4441d2f4b5a45b37bfe77a46d21be4cc9ed","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-21T11:19:04Z","title_canon_sha256":"0b1f7e0f3e30a40abec2e0e96f4a4bbc030561a8c0f6baf8a1f8a968b189a353"},"schema_version":"1.0","source":{"id":"2507.15507","kind":"arxiv","version":1}},"canonical_sha256":"9ba2c5674b79bbcfb0ecb528120e6bba3346251ba6451e1be15f8d329b734929","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9ba2c5674b79bbcfb0ecb528120e6bba3346251ba6451e1be15f8d329b734929","first_computed_at":"2026-07-05T11:40:38.321694Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:40:38.321694Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"vCK+HHRZusXo0S4D++iqK5wGstKzcSqsXCpR+k86T+HyabZ3ZkGl/VVeC2fy5N6+FbJFfKBhQajGa9FK16RfDg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:40:38.322087Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.15507","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b4ff5214c83fbaa9378b1874f80271e43764fed156de46c9cba690f3d3dce30d","sha256:7fe6ffb6399778563ef1654c611b434e62cd4cc57f09d944af7c7826ac0cdee7"],"state_sha256":"191b157bc053a3b085666c1607e4ddd3725fa73f695fdf543b1825fc3c5cb41f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"P1tMeG1f6oW3WuTEsVRhdGib62dU7YvkVSxPzqmH0RIKq91+kZrwocaECOenaVPk2O72AedbYc8bt5kTFDZjDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T19:08:32.403988Z","bundle_sha256":"4dcb06381e610d91009b4369629b1dcf4689745fcbb17b0398975bcb8ba9a668"}}