{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:CCICQZADLLMQ4WQREWESOMMP2E","short_pith_number":"pith:CCICQZAD","schema_version":"1.0","canonical_sha256":"10902864035ad90e5a11258927318fd11e169ace5f2c2bd0b0254cb1f2a19709","source":{"kind":"arxiv","id":"2501.00911","version":1},"attestation_state":"computed","paper":{"title":"Aligning LLMs with Domain Invariant Reward Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"David Wu, Sanjiban Choudhury","submitted_at":"2025-01-01T17:58:31Z","abstract_excerpt":"Aligning large language models (LLMs) to human preferences is challenging in domains where preference data is unavailable. We address the problem of learning reward models for such target domains by leveraging feedback collected from simpler source domains, where human preferences are easier to obtain. Our key insight is that, while domains may differ significantly, human preferences convey \\emph{domain-agnostic} concepts that can be effectively captured by a reward model. We propose \\method, a framework that trains domain-invariant reward models by optimizing a dual loss: a domain loss that m"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2501.00911","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-01T17:58:31Z","cross_cats_sorted":[],"title_canon_sha256":"6429c2c73534b89eba69b425793b947f7cc6cd05be8188efb9b4351d0caa28bf","abstract_canon_sha256":"1176ba064c15d37cfbad9bd08c61004b8f9a414ea89f1078b6c0b7eaa5f34b3a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:56:11.940872Z","signature_b64":"wUAsApP5ibNaPbvtOpSMXBnmRTERORusjJAMP8nbAj9hw80wxOzCkRHP17w9Epo3o3/mjlPK8PVoWlhOdSvPCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"10902864035ad90e5a11258927318fd11e169ace5f2c2bd0b0254cb1f2a19709","last_reissued_at":"2026-07-05T09:56:11.940477Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:56:11.940477Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Aligning LLMs with Domain Invariant Reward Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"David Wu, Sanjiban Choudhury","submitted_at":"2025-01-01T17:58:31Z","abstract_excerpt":"Aligning large language models (LLMs) to human preferences is challenging in domains where preference data is unavailable. We address the problem of learning reward models for such target domains by leveraging feedback collected from simpler source domains, where human preferences are easier to obtain. Our key insight is that, while domains may differ significantly, human preferences convey \\emph{domain-agnostic} concepts that can be effectively captured by a reward model. We propose \\method, a framework that trains domain-invariant reward models by optimizing a dual loss: a domain loss that m"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.00911","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.00911/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2501.00911","created_at":"2026-07-05T09:56:11.940533+00:00"},{"alias_kind":"arxiv_version","alias_value":"2501.00911v1","created_at":"2026-07-05T09:56:11.940533+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.00911","created_at":"2026-07-05T09:56:11.940533+00:00"},{"alias_kind":"pith_short_12","alias_value":"CCICQZADLLMQ","created_at":"2026-07-05T09:56:11.940533+00:00"},{"alias_kind":"pith_short_16","alias_value":"CCICQZADLLMQ4WQR","created_at":"2026-07-05T09:56:11.940533+00:00"},{"alias_kind":"pith_short_8","alias_value":"CCICQZAD","created_at":"2026-07-05T09:56:11.940533+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E","json":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E.json","graph_json":"https://pith.science/api/pith-number/CCICQZADLLMQ4WQREWESOMMP2E/graph.json","events_json":"https://pith.science/api/pith-number/CCICQZADLLMQ4WQREWESOMMP2E/events.json","paper":"https://pith.science/paper/CCICQZAD"},"agent_actions":{"view_html":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E","download_json":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E.json","view_paper":"https://pith.science/paper/CCICQZAD","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2501.00911&json=true","fetch_graph":"https://pith.science/api/pith-number/CCICQZADLLMQ4WQREWESOMMP2E/graph.json","fetch_events":"https://pith.science/api/pith-number/CCICQZADLLMQ4WQREWESOMMP2E/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E/action/storage_attestation","attest_author":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E/action/author_attestation","sign_citation":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E/action/citation_signature","submit_replication":"https://pith.science/pith/CCICQZADLLMQ4WQREWESOMMP2E/action/replication_record"}},"created_at":"2026-07-05T09:56:11.940533+00:00","updated_at":"2026-07-05T09:56:11.940533+00:00"}