{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:AW63KK7PHJUPQNHBJ5OKSHXRIQ","short_pith_number":"pith:AW63KK7P","schema_version":"1.0","canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","source":{"kind":"arxiv","id":"2410.12832","version":1},"attestation_state":"computed","paper":{"title":"Generative Reward Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alon Albalak, Chase Blagden, Chelsea Finn, Dakota Mahan, Duy Van Phung, Jan-Philipp Fr\\\"anken, Louis Castricato, Nathan Lile, Rafael Rafailov","submitted_at":"2024-10-02T17:58:39Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has greatly improved the performance of modern Large Language Models (LLMs). The RLHF process is resource-intensive and technically challenging, generally requiring a large collection of human preference labels over model-generated outputs. Reinforcement Learning from AI Feedback (RLAIF) addresses this data collection challenge by leveraging synthetic preferences generated by an LLM. However, recent work has shown that synthetic preferences labels may not align well with human preference judgments. To address this, we propose a hybrid approach "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.12832","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","cross_cats_sorted":[],"title_canon_sha256":"c7ddee6f9ff056b9d7652efebff129208ee1fa0d2c0f410cbe56d8ef3f4a9822","abstract_canon_sha256":"44f61e00ccd5671f702d3fbfe41a9b6fc69c66f6ef2d628c101bf209314c7504"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:51.533903Z","signature_b64":"XpVpzLmNjEa0PkbnVc3bCEhHzwJo2Pw7scXZliOWpzaiGuj1rA6PUa+z1jwB+25c/yWxwFe5Fflsg6VL9XUOCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","last_reissued_at":"2026-07-05T09:21:51.533559Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:51.533559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Generative Reward Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alon Albalak, Chase Blagden, Chelsea Finn, Dakota Mahan, Duy Van Phung, Jan-Philipp Fr\\\"anken, Louis Castricato, Nathan Lile, Rafael Rafailov","submitted_at":"2024-10-02T17:58:39Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has greatly improved the performance of modern Large Language Models (LLMs). The RLHF process is resource-intensive and technically challenging, generally requiring a large collection of human preference labels over model-generated outputs. Reinforcement Learning from AI Feedback (RLAIF) addresses this data collection challenge by leveraging synthetic preferences generated by an LLM. However, recent work has shown that synthetic preferences labels may not align well with human preference judgments. To address this, we propose a hybrid approach "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.12832","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.12832/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.12832","created_at":"2026-07-05T09:21:51.533614+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.12832v1","created_at":"2026-07-05T09:21:51.533614+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.12832","created_at":"2026-07-05T09:21:51.533614+00:00"},{"alias_kind":"pith_short_12","alias_value":"AW63KK7PHJUP","created_at":"2026-07-05T09:21:51.533614+00:00"},{"alias_kind":"pith_short_16","alias_value":"AW63KK7PHJUPQNHB","created_at":"2026-07-05T09:21:51.533614+00:00"},{"alias_kind":"pith_short_8","alias_value":"AW63KK7P","created_at":"2026-07-05T09:21:51.533614+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":22,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25996","citing_title":"Autodata: An agentic data scientist to create high quality synthetic data","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2606.25996","citing_title":"Autodata: An agentic data scientist to create high quality synthetic data","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09932","citing_title":"When RL Fails after SFT: Rejuvenating Model Plasticity for Robust SFT-to-RL Handoff","ref_index":112,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01249","citing_title":"Trust Region On-Policy Distillation","ref_index":154,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30339","citing_title":"REAR: Test-time Preference Realignment through Reward Decomposition","ref_index":131,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29156","citing_title":"RUBRIC-ARROW: Alternating Pointwise Rubric Reward Modeling for LLM Post-training in Non-verifiable Domains","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00609","citing_title":"CARE-RL: Capability-Aware Reinforcement Learning for Mitigating Cross-Domain Conflicts","ref_index":63,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20745","citing_title":"The Hidden Signal of Verifier Strictness: Controlling and Improving Step-Wise Verification via Selective Latent Steering","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2506.01937","citing_title":"RewardBench 2: Advancing Reward Model Evaluation","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2509.00084","citing_title":"Learning to Refine: Self-Refinement of Parallel Reasoning in LLMs","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2510.14232","citing_title":"Scaling Test-Time Compute to Achieve IOI Gold Medal with Open-Weight Models","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2510.24235","citing_title":"PaTaRM: Bridging Pairwise and Pointwise Signals via Preference-Aware Task-Adaptive Reward Modeling","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16335","citing_title":"Beyond Verifiable Rewards: Rubric-Based GRM for Reinforced Fine-Tuning SWE Agents","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01831","citing_title":"RMGAP: Benchmarking the Generalization of Reward Models across Diverse Preferences","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10701","citing_title":"Bringing Value Models Back: Generative Critics for Value Modeling in LLM Reinforcement Learning","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11611","citing_title":"Utilizing and Calibrating Hindsight Process Rewards via Reinforcement with Mutual Information Self-Evaluation","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11626","citing_title":"RationalRewards: Reasoning Rewards Scale Visual Generation Both Training and Test Time","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.07506","citing_title":"ReflectRM: Boosting Generative Reward Models via Self-Reflection within a Unified Judgment Framework","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2505.07062","citing_title":"Seed1.5-VL Technical Report","ref_index":87,"is_internal_anchor":false},{"citing_arxiv_id":"2604.07484","citing_title":"ConsistRM: Improving Generative Reward Models via Consistency-Aware Self-Training","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2604.05517","citing_title":"UniCreative: Unifying Long-form Logic and Short-form Sparkle via Reference-Free Reinforcement Learning","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13602","citing_title":"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges","ref_index":133,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ","json":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ.json","graph_json":"https://pith.science/api/pith-number/AW63KK7PHJUPQNHBJ5OKSHXRIQ/graph.json","events_json":"https://pith.science/api/pith-number/AW63KK7PHJUPQNHBJ5OKSHXRIQ/events.json","paper":"https://pith.science/paper/AW63KK7P"},"agent_actions":{"view_html":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ","download_json":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ.json","view_paper":"https://pith.science/paper/AW63KK7P","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.12832&json=true","fetch_graph":"https://pith.science/api/pith-number/AW63KK7PHJUPQNHBJ5OKSHXRIQ/graph.json","fetch_events":"https://pith.science/api/pith-number/AW63KK7PHJUPQNHBJ5OKSHXRIQ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/action/storage_attestation","attest_author":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/action/author_attestation","sign_citation":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/action/citation_signature","submit_replication":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/action/replication_record"}},"created_at":"2026-07-05T09:21:51.533614+00:00","updated_at":"2026-07-05T09:21:51.533614+00:00"}