{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VBRFHVEDZ4KT77SMSN6K3HGVTT","short_pith_number":"pith:VBRFHVED","canonical_record":{"source":{"id":"2409.20370","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-30T15:06:53Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"c80d1d26af885353df0895763e9e3cccc5fa5f074ed3afa8fdd2b7b77e5ada49","abstract_canon_sha256":"8eba856c5c8efc193f1001c24dcccc2a73f32308d414e5b9dd3d23738587e88b"},"schema_version":"1.0"},"canonical_sha256":"a86253d483cf153ffe4c937cad9cd59ce22414be15d2824e93e4d14a52dee964","source":{"kind":"arxiv","id":"2409.20370","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.20370","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"arxiv_version","alias_value":"2409.20370v1","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.20370","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_12","alias_value":"VBRFHVEDZ4KT","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_16","alias_value":"VBRFHVEDZ4KT77SM","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_8","alias_value":"VBRFHVED","created_at":"2026-07-05T09:13:42Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VBRFHVEDZ4KT77SMSN6K3HGVTT","target":"record","payload":{"canonical_record":{"source":{"id":"2409.20370","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-30T15:06:53Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"c80d1d26af885353df0895763e9e3cccc5fa5f074ed3afa8fdd2b7b77e5ada49","abstract_canon_sha256":"8eba856c5c8efc193f1001c24dcccc2a73f32308d414e5b9dd3d23738587e88b"},"schema_version":"1.0"},"canonical_sha256":"a86253d483cf153ffe4c937cad9cd59ce22414be15d2824e93e4d14a52dee964","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:13:42.498753Z","signature_b64":"Bk9twLGGgqyvBvoRIWeNgaB1Uwzp8yT4n3KCdwfqdb0Up4x4TW+3YWXZT6MLGWh/mdIIPZDC4DXNnrXtm4gJDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a86253d483cf153ffe4c937cad9cd59ce22414be15d2824e93e4d14a52dee964","last_reissued_at":"2026-07-05T09:13:42.498279Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:13:42.498279Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.20370","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:13:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D+VvyGl+/u7ao63BaHPfTSCSvPAvWUjeeIsF3TWvEicbnZ5FrkKdkDVczeaM5BwDdIJfj7XS8fbX3FACrkMPDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T13:47:47.320989Z"},"content_sha256":"a341fcaed531249d89f9c6394dad4f9cb6fd1622ed8cb2bebbec5829cdeef9b1","schema_version":"1.0","event_id":"sha256:a341fcaed531249d89f9c6394dad4f9cb6fd1622ed8cb2bebbec5829cdeef9b1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VBRFHVEDZ4KT77SMSN6K3HGVTT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Perfect Blend: Redefining RLHF with Mixture of Judges","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Arya Talabzadeh, Chen Zhu, Di Jin, Eric Han, Eryk Helenowski, Gabriel Cohen, Han Fang, Hao Ma, Hejia Zhang, Kaiyan Peng, Karishma Mandyam, Karthik Abinav Sankararaman, Madian Khabsa, Shaoliang Nie, Sinong Wang, Tengyu Xu, Wenxuan Zhou, Yuandong Tian, Yun He, Zhouhao Zeng","submitted_at":"2024-09-30T15:06:53Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has become the leading approach for fine-tuning large language models (LLM). However, RLHF has limitations in multi-task learning (MTL) due to challenges of reward hacking and extreme multi-objective optimization (i.e., trade-off of multiple and/or sometimes conflicting objectives). Applying RLHF for MTL currently requires careful tuning of the weights for reward model and data combinations. This is often done via human intuition and does not generalize. In this work, we introduce a novel post-training paradigm which we called Constrained Gener"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.20370","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.20370/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:13:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8laaafET7BTrPBi3ndTpS0yfuMk1XTCZ3MzDFOvD2+t0gw6rjgICB6sNO4uTW8G6M3CQ236xBNeHSPS9MHeWAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T13:47:47.321373Z"},"content_sha256":"993b48fdb7e2939b11cda3723fb6b2f410cb9753067f179be539213ba0bf8b2c","schema_version":"1.0","event_id":"sha256:993b48fdb7e2939b11cda3723fb6b2f410cb9753067f179be539213ba0bf8b2c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/bundle.json","state_url":"https://pith.science/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T13:47:47Z","links":{"resolver":"https://pith.science/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT","bundle":"https://pith.science/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/bundle.json","state":"https://pith.science/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VBRFHVEDZ4KT77SMSN6K3HGVTT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VBRFHVEDZ4KT77SMSN6K3HGVTT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8eba856c5c8efc193f1001c24dcccc2a73f32308d414e5b9dd3d23738587e88b","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-30T15:06:53Z","title_canon_sha256":"c80d1d26af885353df0895763e9e3cccc5fa5f074ed3afa8fdd2b7b77e5ada49"},"schema_version":"1.0","source":{"id":"2409.20370","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.20370","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"arxiv_version","alias_value":"2409.20370v1","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.20370","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_12","alias_value":"VBRFHVEDZ4KT","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_16","alias_value":"VBRFHVEDZ4KT77SM","created_at":"2026-07-05T09:13:42Z"},{"alias_kind":"pith_short_8","alias_value":"VBRFHVED","created_at":"2026-07-05T09:13:42Z"}],"graph_snapshots":[{"event_id":"sha256:993b48fdb7e2939b11cda3723fb6b2f410cb9753067f179be539213ba0bf8b2c","target":"graph","created_at":"2026-07-05T09:13:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.20370/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has become the leading approach for fine-tuning large language models (LLM). However, RLHF has limitations in multi-task learning (MTL) due to challenges of reward hacking and extreme multi-objective optimization (i.e., trade-off of multiple and/or sometimes conflicting objectives). Applying RLHF for MTL currently requires careful tuning of the weights for reward model and data combinations. This is often done via human intuition and does not generalize. In this work, we introduce a novel post-training paradigm which we called Constrained Gener","authors_text":"Arya Talabzadeh, Chen Zhu, Di Jin, Eric Han, Eryk Helenowski, Gabriel Cohen, Han Fang, Hao Ma, Hejia Zhang, Kaiyan Peng, Karishma Mandyam, Karthik Abinav Sankararaman, Madian Khabsa, Shaoliang Nie, Sinong Wang, Tengyu Xu, Wenxuan Zhou, Yuandong Tian, Yun He, Zhouhao Zeng","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-30T15:06:53Z","title":"The Perfect Blend: Redefining RLHF with Mixture of Judges"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.20370","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a341fcaed531249d89f9c6394dad4f9cb6fd1622ed8cb2bebbec5829cdeef9b1","target":"record","created_at":"2026-07-05T09:13:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8eba856c5c8efc193f1001c24dcccc2a73f32308d414e5b9dd3d23738587e88b","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-30T15:06:53Z","title_canon_sha256":"c80d1d26af885353df0895763e9e3cccc5fa5f074ed3afa8fdd2b7b77e5ada49"},"schema_version":"1.0","source":{"id":"2409.20370","kind":"arxiv","version":1}},"canonical_sha256":"a86253d483cf153ffe4c937cad9cd59ce22414be15d2824e93e4d14a52dee964","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a86253d483cf153ffe4c937cad9cd59ce22414be15d2824e93e4d14a52dee964","first_computed_at":"2026-07-05T09:13:42.498279Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:13:42.498279Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Bk9twLGGgqyvBvoRIWeNgaB1Uwzp8yT4n3KCdwfqdb0Up4x4TW+3YWXZT6MLGWh/mdIIPZDC4DXNnrXtm4gJDw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:13:42.498753Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.20370","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a341fcaed531249d89f9c6394dad4f9cb6fd1622ed8cb2bebbec5829cdeef9b1","sha256:993b48fdb7e2939b11cda3723fb6b2f410cb9753067f179be539213ba0bf8b2c"],"state_sha256":"b88a5bf13ee1d54f43fd3d13a7a29ca99b8433b020349765349c22203284c32c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KE6Z8/r1G25m0DBv9wbr9gENjuKnGLLrmrFv77CHFSAY/QFxn4ydVPpGgTUOIhxbK4gruGvk35mOLNEGa59cDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T13:47:47.323662Z","bundle_sha256":"375c959d775197c51338c47dd88cff9ba503b0d35f47e0aa64a2fb615f76adc1"}}