{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:J37KW4IQF25KHSSSFQONMAMNID","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7bb4e47ab64e8e5393e9239ecf4fa1ce61e861dd66687b21c0ba42120d8ac482","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-11T14:24:00Z","title_canon_sha256":"57abf7b6a6847f0b7c18cceb6173f72d4434276742725eb335bac8fcaf0ccf6d"},"schema_version":"1.0","source":{"id":"2406.07295","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.07295","created_at":"2026-07-05T08:30:52Z"},{"alias_kind":"arxiv_version","alias_value":"2406.07295v2","created_at":"2026-07-05T08:30:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.07295","created_at":"2026-07-05T08:30:52Z"},{"alias_kind":"pith_short_12","alias_value":"J37KW4IQF25K","created_at":"2026-07-05T08:30:52Z"},{"alias_kind":"pith_short_16","alias_value":"J37KW4IQF25KHSSS","created_at":"2026-07-05T08:30:52Z"},{"alias_kind":"pith_short_8","alias_value":"J37KW4IQ","created_at":"2026-07-05T08:30:52Z"}],"graph_snapshots":[{"event_id":"sha256:c128a6d6473281d37aa1807609088270689df15a630b739ac15f1e6e3858d4a2","target":"graph","created_at":"2026-07-05T08:30:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.07295/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper presents Multi-Objective Reinforcement Learning from AI Feedback (MORLAIF), a novel approach to improving the alignment and performance of language models trained using reinforcement learning from AI feedback (RLAIF). In contrast to standard approaches that train a single preference model to represent all human preferences, MORLAIF decomposes this task into multiple simpler principles, such as toxicity, factuality, and sycophancy. Separate preference models are trained for each principle using feedback from GPT-3.5-Turbo. These preference model scores are then combined using differe","authors_text":"Marcus Williams","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-11T14:24:00Z","title":"Multi-objective Reinforcement learning from AI Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.07295","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0cfc0d9cc0f902a01859ed199e386e34b9f895f342a2342648a81ea3b2386b48","target":"record","created_at":"2026-07-05T08:30:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7bb4e47ab64e8e5393e9239ecf4fa1ce61e861dd66687b21c0ba42120d8ac482","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-11T14:24:00Z","title_canon_sha256":"57abf7b6a6847f0b7c18cceb6173f72d4434276742725eb335bac8fcaf0ccf6d"},"schema_version":"1.0","source":{"id":"2406.07295","kind":"arxiv","version":2}},"canonical_sha256":"4efeab71102ebaa3ca522c1cd6018d40ec58533bf6825e9a78b5d74e4f59aa66","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4efeab71102ebaa3ca522c1cd6018d40ec58533bf6825e9a78b5d74e4f59aa66","first_computed_at":"2026-07-05T08:30:52.705999Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:30:52.705999Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"pCm9qGK7eI7vpvcUaHKnh8+wV2GL+5KW7ASgdMSZ0cVa49HK3ZvpcrHclO6HN2B0UymLEIdjyS/oGS2m17EOCw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:30:52.706467Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.07295","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0cfc0d9cc0f902a01859ed199e386e34b9f895f342a2342648a81ea3b2386b48","sha256:c128a6d6473281d37aa1807609088270689df15a630b739ac15f1e6e3858d4a2"],"state_sha256":"c51dda82657c0478d36992da65dc7bf6a16352c0304cd3b97cfbf9e682e41288"}