{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:XBP5CBPC5NOBOJPEGDOOFJVZFJ","short_pith_number":"pith:XBP5CBPC","schema_version":"1.0","canonical_sha256":"b85fd105e2eb5c1725e430dce2a6b92a7744256222a43e21915f20bce7e4eba9","source":{"kind":"arxiv","id":"2605.30888","version":1},"attestation_state":"computed","paper":{"title":"The Flip Side of RLHF: On-Policy Feedback for Reward Model Self-Supervised Improvement","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jiaqi Li, Min Tang, Qi Liu, Tong Wu, Xiaobo Wang, Zilong Zheng","submitted_at":"2026-05-29T06:25:16Z","abstract_excerpt":"Building strong reward models (RMs) for language model alignment is bottlenecked by the cost and difficulty of acquiring diverse and reliable preference data from human annotation or judge models. It is dramatically worse as the policy evolves beyond the static RM training. Therefore, we propose SAVE (Self-supervised reward model improvement via Value-Anchored On-policy feedback), a framework that grades on-policy responses as feedback by using the value function for on-policy RM training. SAVE naturally converts the reward-graded on-policy responses into supervision with a prompt-specific val"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2605.30888","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-05-29T06:25:16Z","cross_cats_sorted":[],"title_canon_sha256":"bbc9e2ee339a6f2ea3c4d91cbf7fd694f8ca2826a10f25fe68f4f2da07753bfc","abstract_canon_sha256":"8c9992854c0871af57a7263b526968a6ae3be8e9483b66796fe09f9f5659cc75"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-01T01:03:23.354001Z","signature_b64":"bzmBd2+p1tmjs9ApWKBPz68SqMdxQ6Iz25Gxad5HYhIH+8TosMKdFl2r6jsTLEjIhJrmJEfSVxCf6FcxnYxgAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b85fd105e2eb5c1725e430dce2a6b92a7744256222a43e21915f20bce7e4eba9","last_reissued_at":"2026-06-01T01:03:23.353029Z","signature_status":"signed_v1","first_computed_at":"2026-06-01T01:03:23.353029Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Flip Side of RLHF: On-Policy Feedback for Reward Model Self-Supervised Improvement","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jiaqi Li, Min Tang, Qi Liu, Tong Wu, Xiaobo Wang, Zilong Zheng","submitted_at":"2026-05-29T06:25:16Z","abstract_excerpt":"Building strong reward models (RMs) for language model alignment is bottlenecked by the cost and difficulty of acquiring diverse and reliable preference data from human annotation or judge models. It is dramatically worse as the policy evolves beyond the static RM training. Therefore, we propose SAVE (Self-supervised reward model improvement via Value-Anchored On-policy feedback), a framework that grades on-policy responses as feedback by using the value function for on-policy RM training. SAVE naturally converts the reward-graded on-policy responses into supervision with a prompt-specific val"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2605.30888","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2605.30888/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2605.30888","created_at":"2026-06-01T01:03:23.353178+00:00"},{"alias_kind":"arxiv_version","alias_value":"2605.30888v1","created_at":"2026-06-01T01:03:23.353178+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.30888","created_at":"2026-06-01T01:03:23.353178+00:00"},{"alias_kind":"pith_short_12","alias_value":"XBP5CBPC5NOB","created_at":"2026-06-01T01:03:23.353178+00:00"},{"alias_kind":"pith_short_16","alias_value":"XBP5CBPC5NOBOJPE","created_at":"2026-06-01T01:03:23.353178+00:00"},{"alias_kind":"pith_short_8","alias_value":"XBP5CBPC","created_at":"2026-06-01T01:03:23.353178+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ","json":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ.json","graph_json":"https://pith.science/api/pith-number/XBP5CBPC5NOBOJPEGDOOFJVZFJ/graph.json","events_json":"https://pith.science/api/pith-number/XBP5CBPC5NOBOJPEGDOOFJVZFJ/events.json","paper":"https://pith.science/paper/XBP5CBPC"},"agent_actions":{"view_html":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ","download_json":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ.json","view_paper":"https://pith.science/paper/XBP5CBPC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2605.30888&json=true","fetch_graph":"https://pith.science/api/pith-number/XBP5CBPC5NOBOJPEGDOOFJVZFJ/graph.json","fetch_events":"https://pith.science/api/pith-number/XBP5CBPC5NOBOJPEGDOOFJVZFJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ/action/storage_attestation","attest_author":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ/action/author_attestation","sign_citation":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ/action/citation_signature","submit_replication":"https://pith.science/pith/XBP5CBPC5NOBOJPEGDOOFJVZFJ/action/replication_record"}},"created_at":"2026-06-01T01:03:23.353178+00:00","updated_at":"2026-06-01T01:03:23.353178+00:00"}