{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MNTG64K7BXBTAMK7RR4NTSGO2M","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"33a02002a7dd0c0bfb54d5295a0b56969710990f968c2196800fc3f422e96a39","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-25T18:28:26Z","title_canon_sha256":"d09f4ffcef2ef9bbe1d4380d09494bb8983262d92571898a759884c42adf50d1"},"schema_version":"1.0","source":{"id":"2411.16646","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.16646","created_at":"2026-07-05T10:11:29Z"},{"alias_kind":"arxiv_version","alias_value":"2411.16646v3","created_at":"2026-07-05T10:11:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.16646","created_at":"2026-07-05T10:11:29Z"},{"alias_kind":"pith_short_12","alias_value":"MNTG64K7BXBT","created_at":"2026-07-05T10:11:29Z"},{"alias_kind":"pith_short_16","alias_value":"MNTG64K7BXBTAMK7","created_at":"2026-07-05T10:11:29Z"},{"alias_kind":"pith_short_8","alias_value":"MNTG64K7","created_at":"2026-07-05T10:11:29Z"}],"graph_snapshots":[{"event_id":"sha256:a0f82b8d9609d3a1fa471751193f861d389cea46e7eb0af1dfe155d53376871f","target":"graph","created_at":"2026-07-05T10:11:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.16646/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reward modeling is crucial for aligning large language models (LLMs) with human preferences, especially in reinforcement learning from human feedback (RLHF). However, current reward models mainly produce scalar scores and struggle to incorporate critiques in a natural language format. We hypothesize that predicting both critiques and the scalar reward would improve reward modeling ability. Motivated by this, we propose Critic-RM, a framework that improves reward models using self-generated critiques without extra supervision. Critic-RM employs a two-stage process: generating and filtering high","authors_text":"Aston Zhang, Chao Zhang, Chenguang Zhu, Dhruv Mahajan, Liang Tan, Melanie Kambadur, Richard Yuanzhe Pang, Rui Hou, Suchin Gururangan, Xuewei Wang, Yue Yu, Yundi Qian, Zhengxing Chen","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-25T18:28:26Z","title":"Self-Generated Critiques Boost Reward Modeling for Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.16646","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6b03bbe8fe89bf902bd71b62d2f436d8af653564ab8f16ae06b7d8c9a7bd50c5","target":"record","created_at":"2026-07-05T10:11:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"33a02002a7dd0c0bfb54d5295a0b56969710990f968c2196800fc3f422e96a39","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-25T18:28:26Z","title_canon_sha256":"d09f4ffcef2ef9bbe1d4380d09494bb8983262d92571898a759884c42adf50d1"},"schema_version":"1.0","source":{"id":"2411.16646","kind":"arxiv","version":3}},"canonical_sha256":"63666f715f0dc330315f8c78d9c8ced3248cb6974fea87b6ff1162795742357f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"63666f715f0dc330315f8c78d9c8ced3248cb6974fea87b6ff1162795742357f","first_computed_at":"2026-07-05T10:11:29.140925Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:11:29.140925Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"aEvsrChwjez7wrUUYk9ikcHuPWZqfMQ4EaqvPs7U4/B/WcflshdcklSuOPA6ipXiDj4BPeGJz92K2Z1BFwN4Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:11:29.141394Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.16646","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6b03bbe8fe89bf902bd71b62d2f436d8af653564ab8f16ae06b7d8c9a7bd50c5","sha256:a0f82b8d9609d3a1fa471751193f861d389cea46e7eb0af1dfe155d53376871f"],"state_sha256":"53000845fbbc061564ca552a08615f08c587b7b0804c64f4d795f409735689cd"}