{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:PHQQ6W26XOY2D2BL6J7QWSQBJL","short_pith_number":"pith:PHQQ6W26","schema_version":"1.0","canonical_sha256":"79e10f5b5ebbb1a1e82bf27f0b4a014af6b4a96aa66c6186f0d516fdeec7bf21","source":{"kind":"arxiv","id":"2411.01111","version":1},"attestation_state":"computed","paper":{"title":"Rule Based Rewards for Language Model Safety","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Alec Helyar, Alex Beutel, Andrea Vallone, Ian Kivlichan, Johannes Heidecke, John Schulman, Joshua Achiam, Lilian Weng, Molly Lin, Tong Mu","submitted_at":"2024-11-02T02:22:21Z","abstract_excerpt":"Reinforcement learning based fine-tuning of large language models (LLMs) on human preferences has been shown to enhance both their capabilities and safety behavior. However, in cases related to safety, without precise instructions to human annotators, the data collected may cause the model to become overly cautious, or to respond in an undesirable style, such as being judgmental. Additionally, as model capabilities and usage patterns evolve, there may be a costly need to add or relabel data to modify safety behavior. We propose a novel preference modeling approach that utilizes AI feedback and"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.01111","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-11-02T02:22:21Z","cross_cats_sorted":[],"title_canon_sha256":"712aa99b70285964f2d87066f20802020e775d1d71da8e75bbed74f37a41a0b0","abstract_canon_sha256":"05d6fc19b4dea9acdd4ebc65022ee98a927820abea0590b69560ca004eb879bc"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:30:24.868545Z","signature_b64":"yXzOoaobztNejStOg7zdzXT/OiDBJxnEVtwpWKJyH4wg92ImI88naPVUNNBv12MXFSkYkhpaNFBxxs2WJuCUBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"79e10f5b5ebbb1a1e82bf27f0b4a014af6b4a96aa66c6186f0d516fdeec7bf21","last_reissued_at":"2026-07-05T09:30:24.868031Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:30:24.868031Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Rule Based Rewards for Language Model Safety","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Alec Helyar, Alex Beutel, Andrea Vallone, Ian Kivlichan, Johannes Heidecke, John Schulman, Joshua Achiam, Lilian Weng, Molly Lin, Tong Mu","submitted_at":"2024-11-02T02:22:21Z","abstract_excerpt":"Reinforcement learning based fine-tuning of large language models (LLMs) on human preferences has been shown to enhance both their capabilities and safety behavior. However, in cases related to safety, without precise instructions to human annotators, the data collected may cause the model to become overly cautious, or to respond in an undesirable style, such as being judgmental. Additionally, as model capabilities and usage patterns evolve, there may be a costly need to add or relabel data to modify safety behavior. We propose a novel preference modeling approach that utilizes AI feedback and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.01111","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.01111/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.01111","created_at":"2026-07-05T09:30:24.868122+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.01111v1","created_at":"2026-07-05T09:30:24.868122+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.01111","created_at":"2026-07-05T09:30:24.868122+00:00"},{"alias_kind":"pith_short_12","alias_value":"PHQQ6W26XOY2","created_at":"2026-07-05T09:30:24.868122+00:00"},{"alias_kind":"pith_short_16","alias_value":"PHQQ6W26XOY2D2BL","created_at":"2026-07-05T09:30:24.868122+00:00"},{"alias_kind":"pith_short_8","alias_value":"PHQQ6W26","created_at":"2026-07-05T09:30:24.868122+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2509.21319","citing_title":"RLBFF: Binary Flexible Feedback to bridge between Human Feedback & Verifiable Rewards","ref_index":24,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL","json":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL.json","graph_json":"https://pith.science/api/pith-number/PHQQ6W26XOY2D2BL6J7QWSQBJL/graph.json","events_json":"https://pith.science/api/pith-number/PHQQ6W26XOY2D2BL6J7QWSQBJL/events.json","paper":"https://pith.science/paper/PHQQ6W26"},"agent_actions":{"view_html":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL","download_json":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL.json","view_paper":"https://pith.science/paper/PHQQ6W26","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.01111&json=true","fetch_graph":"https://pith.science/api/pith-number/PHQQ6W26XOY2D2BL6J7QWSQBJL/graph.json","fetch_events":"https://pith.science/api/pith-number/PHQQ6W26XOY2D2BL6J7QWSQBJL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL/action/storage_attestation","attest_author":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL/action/author_attestation","sign_citation":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL/action/citation_signature","submit_replication":"https://pith.science/pith/PHQQ6W26XOY2D2BL6J7QWSQBJL/action/replication_record"}},"created_at":"2026-07-05T09:30:24.868122+00:00","updated_at":"2026-07-05T09:30:24.868122+00:00"}