{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AW63KK7PHJUPQNHBJ5OKSHXRIQ","short_pith_number":"pith:AW63KK7P","canonical_record":{"source":{"id":"2410.12832","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","cross_cats_sorted":[],"title_canon_sha256":"c7ddee6f9ff056b9d7652efebff129208ee1fa0d2c0f410cbe56d8ef3f4a9822","abstract_canon_sha256":"44f61e00ccd5671f702d3fbfe41a9b6fc69c66f6ef2d628c101bf209314c7504"},"schema_version":"1.0"},"canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","source":{"kind":"arxiv","id":"2410.12832","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.12832","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"arxiv_version","alias_value":"2410.12832v1","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.12832","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_12","alias_value":"AW63KK7PHJUP","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_16","alias_value":"AW63KK7PHJUPQNHB","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_8","alias_value":"AW63KK7P","created_at":"2026-07-05T09:21:51Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AW63KK7PHJUPQNHBJ5OKSHXRIQ","target":"record","payload":{"canonical_record":{"source":{"id":"2410.12832","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","cross_cats_sorted":[],"title_canon_sha256":"c7ddee6f9ff056b9d7652efebff129208ee1fa0d2c0f410cbe56d8ef3f4a9822","abstract_canon_sha256":"44f61e00ccd5671f702d3fbfe41a9b6fc69c66f6ef2d628c101bf209314c7504"},"schema_version":"1.0"},"canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:51.533903Z","signature_b64":"XpVpzLmNjEa0PkbnVc3bCEhHzwJo2Pw7scXZliOWpzaiGuj1rA6PUa+z1jwB+25c/yWxwFe5Fflsg6VL9XUOCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","last_reissued_at":"2026-07-05T09:21:51.533559Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:51.533559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.12832","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T5CiKqreVxMlctDl6TYZvpo8BWmpYEyaUTfSz3+zRyXWNKiy89V8jDH7IoCMrb57F287jwolnNyLprxcwWivDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T16:59:22.536105Z"},"content_sha256":"54e78b56f0766f1ca1e1cc49a618901945bdc2ad460203c32e1a28654e6c93c0","schema_version":"1.0","event_id":"sha256:54e78b56f0766f1ca1e1cc49a618901945bdc2ad460203c32e1a28654e6c93c0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AW63KK7PHJUPQNHBJ5OKSHXRIQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Generative Reward Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alon Albalak, Chase Blagden, Chelsea Finn, Dakota Mahan, Duy Van Phung, Jan-Philipp Fr\\\"anken, Louis Castricato, Nathan Lile, Rafael Rafailov","submitted_at":"2024-10-02T17:58:39Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has greatly improved the performance of modern Large Language Models (LLMs). The RLHF process is resource-intensive and technically challenging, generally requiring a large collection of human preference labels over model-generated outputs. Reinforcement Learning from AI Feedback (RLAIF) addresses this data collection challenge by leveraging synthetic preferences generated by an LLM. However, recent work has shown that synthetic preferences labels may not align well with human preference judgments. To address this, we propose a hybrid approach "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.12832","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.12832/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oglsmpYuNysv7iuuxkJWysNOVyC9jjXzq2VZORc93iqH28CnxOFsoY4Dt5vII4UI8tc7BEd5XWvTm/8OuIi4Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T16:59:22.536489Z"},"content_sha256":"d10bfb1b8eb7427a8eee4850a03aa3ba3906d095249926c2ab62f61a197a184c","schema_version":"1.0","event_id":"sha256:d10bfb1b8eb7427a8eee4850a03aa3ba3906d095249926c2ab62f61a197a184c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/bundle.json","state_url":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T16:59:22Z","links":{"resolver":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ","bundle":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/bundle.json","state":"https://pith.science/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AW63KK7PHJUPQNHBJ5OKSHXRIQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AW63KK7PHJUPQNHBJ5OKSHXRIQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"44f61e00ccd5671f702d3fbfe41a9b6fc69c66f6ef2d628c101bf209314c7504","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","title_canon_sha256":"c7ddee6f9ff056b9d7652efebff129208ee1fa0d2c0f410cbe56d8ef3f4a9822"},"schema_version":"1.0","source":{"id":"2410.12832","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.12832","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"arxiv_version","alias_value":"2410.12832v1","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.12832","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_12","alias_value":"AW63KK7PHJUP","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_16","alias_value":"AW63KK7PHJUPQNHB","created_at":"2026-07-05T09:21:51Z"},{"alias_kind":"pith_short_8","alias_value":"AW63KK7P","created_at":"2026-07-05T09:21:51Z"}],"graph_snapshots":[{"event_id":"sha256:d10bfb1b8eb7427a8eee4850a03aa3ba3906d095249926c2ab62f61a197a184c","target":"graph","created_at":"2026-07-05T09:21:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.12832/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has greatly improved the performance of modern Large Language Models (LLMs). The RLHF process is resource-intensive and technically challenging, generally requiring a large collection of human preference labels over model-generated outputs. Reinforcement Learning from AI Feedback (RLAIF) addresses this data collection challenge by leveraging synthetic preferences generated by an LLM. However, recent work has shown that synthetic preferences labels may not align well with human preference judgments. To address this, we propose a hybrid approach ","authors_text":"Alon Albalak, Chase Blagden, Chelsea Finn, Dakota Mahan, Duy Van Phung, Jan-Philipp Fr\\\"anken, Louis Castricato, Nathan Lile, Rafael Rafailov","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","title":"Generative Reward Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.12832","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:54e78b56f0766f1ca1e1cc49a618901945bdc2ad460203c32e1a28654e6c93c0","target":"record","created_at":"2026-07-05T09:21:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"44f61e00ccd5671f702d3fbfe41a9b6fc69c66f6ef2d628c101bf209314c7504","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-02T17:58:39Z","title_canon_sha256":"c7ddee6f9ff056b9d7652efebff129208ee1fa0d2c0f410cbe56d8ef3f4a9822"},"schema_version":"1.0","source":{"id":"2410.12832","kind":"arxiv","version":1}},"canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"05bdb52bef3a68f834e14f5ca91ef1442e034252f862a9ceffcfb595df86dec3","first_computed_at":"2026-07-05T09:21:51.533559Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:21:51.533559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XpVpzLmNjEa0PkbnVc3bCEhHzwJo2Pw7scXZliOWpzaiGuj1rA6PUa+z1jwB+25c/yWxwFe5Fflsg6VL9XUOCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:21:51.533903Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.12832","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:54e78b56f0766f1ca1e1cc49a618901945bdc2ad460203c32e1a28654e6c93c0","sha256:d10bfb1b8eb7427a8eee4850a03aa3ba3906d095249926c2ab62f61a197a184c"],"state_sha256":"0e5cc6aa8de073950d6c37045aee04d0de89593c96fcc38fb1c04d9fd7ae646b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"F5g6C9hc4YKxtpHpciyYzRrdjCkWVqx5YEzf8Z1bKOkmiwewOhJpkl60Rw1k3XgyeWZ38RSKP8R8B87RcWnSAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T16:59:22.538459Z","bundle_sha256":"deb66d69298506e39152d6ed34bc36a8c79e6b300bc2bbe241dff65b9be194b2"}}