{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:BXHZNXDFVPKT6TWKSSG3SVGIY4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2c1a4d318430db92cf3e5961b3564ec38d40211e500fd554908573882ee3ee20","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-17T04:34:23Z","title_canon_sha256":"3eeb84b985e421631299fe8542b4389f9bd99205f022b1e9a8e7d227fef9e11d"},"schema_version":"1.0","source":{"id":"2506.06292","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.06292","created_at":"2026-07-05T11:18:43Z"},{"alias_kind":"arxiv_version","alias_value":"2506.06292v2","created_at":"2026-07-05T11:18:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.06292","created_at":"2026-07-05T11:18:43Z"},{"alias_kind":"pith_short_12","alias_value":"BXHZNXDFVPKT","created_at":"2026-07-05T11:18:43Z"},{"alias_kind":"pith_short_16","alias_value":"BXHZNXDFVPKT6TWK","created_at":"2026-07-05T11:18:43Z"},{"alias_kind":"pith_short_8","alias_value":"BXHZNXDF","created_at":"2026-07-05T11:18:43Z"}],"graph_snapshots":[{"event_id":"sha256:45fcf44844c9068655403870d726451692b90cbc1c1d16f7e6014eadfc75f914","target":"graph","created_at":"2026-07-05T11:18:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.06292/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"During the preference optimization of large language models (LLMs), distribution shifts may arise between newly generated model samples and the data used to train the reward model (RM). This shift reduces the efficacy of the RM, which in turn negatively impacts the performance of the policy model (PM). To address this challenge, we propose Mutual-Taught, a self-training method that iteratively improves both the PM and RM without requiring additional human annotation. Our approach mirrors the expectation-maximization (EM) algorithm. In the E-step, the PM is updated using feedback from the curre","authors_text":"Canbin Huang, Fanqi Wan, Longguang Zhong, Ming Yan, Tianyuan Shi, Weizhou Shen, Xiaojun Quan, Ziyi Yang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-17T04:34:23Z","title":"Mutual-Taught for Co-adapting Policy and Reward Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.06292","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:70d4dd1605bdea3ed8bcc57b02fc53d3dadef9842cfec42ee30aac31c98807c0","target":"record","created_at":"2026-07-05T11:18:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2c1a4d318430db92cf3e5961b3564ec38d40211e500fd554908573882ee3ee20","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-17T04:34:23Z","title_canon_sha256":"3eeb84b985e421631299fe8542b4389f9bd99205f022b1e9a8e7d227fef9e11d"},"schema_version":"1.0","source":{"id":"2506.06292","kind":"arxiv","version":2}},"canonical_sha256":"0dcf96dc65abd53f4eca948db954c8c7132c6a552e0cf3e4d619986b21927a20","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0dcf96dc65abd53f4eca948db954c8c7132c6a552e0cf3e4d619986b21927a20","first_computed_at":"2026-07-05T11:18:43.653090Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:18:43.653090Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2DJ6XGrJ5jO0J8cYC8t1jmB5CDYPOYuF/xG3BkjMXgaxC4ox87Ue+6wW44+MTZL0+Ue7HCRGwfr73UMoyTZ5Aw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:18:43.653553Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.06292","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:70d4dd1605bdea3ed8bcc57b02fc53d3dadef9842cfec42ee30aac31c98807c0","sha256:45fcf44844c9068655403870d726451692b90cbc1c1d16f7e6014eadfc75f914"],"state_sha256":"13cb5d8b864d8cb2c88a23f69dfe35ae0c688e8682f04b13ee4f585ad6b0d065"}