{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:6XYURYCUH2TD5EFW2RWHJTKGHR","short_pith_number":"pith:6XYURYCU","schema_version":"1.0","canonical_sha256":"f5f148e0543ea63e90b6d46c74cd463c6af9f443b2d1318d422f61860b237057","source":{"kind":"arxiv","id":"2405.00438","version":1},"attestation_state":"computed","paper":{"title":"MetaRM: Shifted Distributions Alignment via Meta-Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Enyu Zhou, Haoxiang Jia, Junjie Ye, Limao Xiong, Qi Zhang, Rui Zheng, Shihan Dou, Tao Gui, Tianlong Li, Xin Zhao, Xuanjing Huang, Yan Liu","submitted_at":"2024-05-01T10:43:55Z","abstract_excerpt":"The success of Reinforcement Learning from Human Feedback (RLHF) in language model alignment is critically dependent on the capability of the reward model (RM). However, as the training process progresses, the output distribution of the policy model shifts, leading to the RM's reduced ability to distinguish between responses. This issue is further compounded when the RM, trained on a specific data distribution, struggles to generalize to examples outside of that distribution. These two issues can be united as a challenge posed by the shifted distribution of the environment. To surmount this ch"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.00438","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-01T10:43:55Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"a2243f5aa0cef55c3e1458b62e90b2b38311b08ba61b83111fd9da2ce4a74a23","abstract_canon_sha256":"35e57b9bd93dcdcaeaf7fda66a8234b231a860a85934c649cd80a9237fd88f06"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:14:13.034531Z","signature_b64":"95kesPir0jv/jpZE3OnAOm61C+O0fDzFcQigUvkzotTd3z8p7mL01+vrOCRpnRhQCpB0nFfHJ04egLA/YpJABw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f5f148e0543ea63e90b6d46c74cd463c6af9f443b2d1318d422f61860b237057","last_reissued_at":"2026-07-05T08:14:13.034098Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:14:13.034098Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MetaRM: Shifted Distributions Alignment via Meta-Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Enyu Zhou, Haoxiang Jia, Junjie Ye, Limao Xiong, Qi Zhang, Rui Zheng, Shihan Dou, Tao Gui, Tianlong Li, Xin Zhao, Xuanjing Huang, Yan Liu","submitted_at":"2024-05-01T10:43:55Z","abstract_excerpt":"The success of Reinforcement Learning from Human Feedback (RLHF) in language model alignment is critically dependent on the capability of the reward model (RM). However, as the training process progresses, the output distribution of the policy model shifts, leading to the RM's reduced ability to distinguish between responses. This issue is further compounded when the RM, trained on a specific data distribution, struggles to generalize to examples outside of that distribution. These two issues can be united as a challenge posed by the shifted distribution of the environment. To surmount this ch"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00438","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.00438/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.00438","created_at":"2026-07-05T08:14:13.034153+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.00438v1","created_at":"2026-07-05T08:14:13.034153+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00438","created_at":"2026-07-05T08:14:13.034153+00:00"},{"alias_kind":"pith_short_12","alias_value":"6XYURYCUH2TD","created_at":"2026-07-05T08:14:13.034153+00:00"},{"alias_kind":"pith_short_16","alias_value":"6XYURYCUH2TD5EFW","created_at":"2026-07-05T08:14:13.034153+00:00"},{"alias_kind":"pith_short_8","alias_value":"6XYURYCU","created_at":"2026-07-05T08:14:13.034153+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2502.08922","citing_title":"Self-Consistency of the Internal Reward Models Improves Self-Rewarding Language Models","ref_index":15,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR","json":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR.json","graph_json":"https://pith.science/api/pith-number/6XYURYCUH2TD5EFW2RWHJTKGHR/graph.json","events_json":"https://pith.science/api/pith-number/6XYURYCUH2TD5EFW2RWHJTKGHR/events.json","paper":"https://pith.science/paper/6XYURYCU"},"agent_actions":{"view_html":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR","download_json":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR.json","view_paper":"https://pith.science/paper/6XYURYCU","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.00438&json=true","fetch_graph":"https://pith.science/api/pith-number/6XYURYCUH2TD5EFW2RWHJTKGHR/graph.json","fetch_events":"https://pith.science/api/pith-number/6XYURYCUH2TD5EFW2RWHJTKGHR/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR/action/storage_attestation","attest_author":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR/action/author_attestation","sign_citation":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR/action/citation_signature","submit_replication":"https://pith.science/pith/6XYURYCUH2TD5EFW2RWHJTKGHR/action/replication_record"}},"created_at":"2026-07-05T08:14:13.034153+00:00","updated_at":"2026-07-05T08:14:13.034153+00:00"}