{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:ED2RYF65QGFTRD66ZGFEX24RZJ","short_pith_number":"pith:ED2RYF65","schema_version":"1.0","canonical_sha256":"20f51c17dd818b388fdec98a4beb91ca5108f364cce15a0175022ab02dc2b0a5","source":{"kind":"arxiv","id":"2310.01045","version":2},"attestation_state":"computed","paper":{"title":"Tool-Augmented Reward Modeling","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hao Tian, Hua Wu, Lei Li, Ningyu Zhang, Shuohuan Wang, Yekun Chai, Yu Sun","submitted_at":"2023-10-02T09:47:40Z","abstract_excerpt":"Reward modeling (a.k.a., preference modeling) is instrumental for aligning large language models with human preferences, particularly within the context of reinforcement learning from human feedback (RLHF). While conventional reward models (RMs) have exhibited remarkable scalability, they oft struggle with fundamental functionality such as arithmetic computation, code execution, and factual lookup. In this paper, we propose a tool-augmented preference modeling approach, named Themis, to address these limitations by empowering RMs with access to external environments, including calculators and "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.01045","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-02T09:47:40Z","cross_cats_sorted":[],"title_canon_sha256":"0d4fb0b0c1eca6497debcd8f079e31467e39a5a66410ceb95340b0797facdb1d","abstract_canon_sha256":"490bc3f32088d4624d7c113720c9add78b365af3ab6fa4c2407af354d2ea9b17"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:43:56.241231Z","signature_b64":"YDWAdqGstElNAz9oS6ayFMIC+VZfBgzMUy6vuDA+uw6dnrnPUwsByg+ZmgpT0XEv5L197SWMuBzoT6pL+OlpBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"20f51c17dd818b388fdec98a4beb91ca5108f364cce15a0175022ab02dc2b0a5","last_reissued_at":"2026-07-05T07:43:56.240723Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:43:56.240723Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Tool-Augmented Reward Modeling","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hao Tian, Hua Wu, Lei Li, Ningyu Zhang, Shuohuan Wang, Yekun Chai, Yu Sun","submitted_at":"2023-10-02T09:47:40Z","abstract_excerpt":"Reward modeling (a.k.a., preference modeling) is instrumental for aligning large language models with human preferences, particularly within the context of reinforcement learning from human feedback (RLHF). While conventional reward models (RMs) have exhibited remarkable scalability, they oft struggle with fundamental functionality such as arithmetic computation, code execution, and factual lookup. In this paper, we propose a tool-augmented preference modeling approach, named Themis, to address these limitations by empowering RMs with access to external environments, including calculators and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.01045","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.01045/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.01045","created_at":"2026-07-05T07:43:56.240777+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.01045v2","created_at":"2026-07-05T07:43:56.240777+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.01045","created_at":"2026-07-05T07:43:56.240777+00:00"},{"alias_kind":"pith_short_12","alias_value":"ED2RYF65QGFT","created_at":"2026-07-05T07:43:56.240777+00:00"},{"alias_kind":"pith_short_16","alias_value":"ED2RYF65QGFTRD66","created_at":"2026-07-05T07:43:56.240777+00:00"},{"alias_kind":"pith_short_8","alias_value":"ED2RYF65","created_at":"2026-07-05T07:43:56.240777+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ","json":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ.json","graph_json":"https://pith.science/api/pith-number/ED2RYF65QGFTRD66ZGFEX24RZJ/graph.json","events_json":"https://pith.science/api/pith-number/ED2RYF65QGFTRD66ZGFEX24RZJ/events.json","paper":"https://pith.science/paper/ED2RYF65"},"agent_actions":{"view_html":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ","download_json":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ.json","view_paper":"https://pith.science/paper/ED2RYF65","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.01045&json=true","fetch_graph":"https://pith.science/api/pith-number/ED2RYF65QGFTRD66ZGFEX24RZJ/graph.json","fetch_events":"https://pith.science/api/pith-number/ED2RYF65QGFTRD66ZGFEX24RZJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ/action/storage_attestation","attest_author":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ/action/author_attestation","sign_citation":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ/action/citation_signature","submit_replication":"https://pith.science/pith/ED2RYF65QGFTRD66ZGFEX24RZJ/action/replication_record"}},"created_at":"2026-07-05T07:43:56.240777+00:00","updated_at":"2026-07-05T07:43:56.240777+00:00"}