{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:IANJ4GE27ZDLVG2GRIRNJCVVZG","short_pith_number":"pith:IANJ4GE2","schema_version":"1.0","canonical_sha256":"401a9e189afe46ba9b468a22d48ab5c99060e399a04dc921dd5587c7b4d8c7f7","source":{"kind":"arxiv","id":"1908.04734","version":5},"attestation_state":"computed","paper":{"title":"Reward Tampering Problems and Solutions in Reinforcement Learning: A Causal Influence Diagram Perspective","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Marcus Hutter, Ramana Kumar, Tom Everitt, Victoria Krakovna","submitted_at":"2019-08-13T16:50:00Z","abstract_excerpt":"Can humans get arbitrarily capable reinforcement learning (RL) agents to do their bidding? Or will sufficiently capable RL agents always find ways to bypass their intended objectives by shortcutting their reward signal? This question impacts how far RL can be scaled, and whether alternative paradigms must be developed in order to build safe artificial general intelligence. In this paper, we study when an RL agent has an instrumental goal to tamper with its reward process, and describe design principles that prevent instrumental goals for two different types of reward tampering (reward function"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1908.04734","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2019-08-13T16:50:00Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"da2804d55171c5f9a101ab76526bf8ab9bf04470aa3609d62d248ab8241570ba","abstract_canon_sha256":"87e17e0d31ab2e2b6beb828eaf7e09b5b919a915cf3135c4f8c59daae38be4d7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:26:25.716230Z","signature_b64":"/MI7zTOrY/wBCBOtog0L33Jta0IB2432FdsIFzFkDHwUMfcbqNbs32i7BghrOAklW/il+P+l4xPUootTq5AXAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"401a9e189afe46ba9b468a22d48ab5c99060e399a04dc921dd5587c7b4d8c7f7","last_reissued_at":"2026-07-05T02:26:25.715741Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:26:25.715741Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Reward Tampering Problems and Solutions in Reinforcement Learning: A Causal Influence Diagram Perspective","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Marcus Hutter, Ramana Kumar, Tom Everitt, Victoria Krakovna","submitted_at":"2019-08-13T16:50:00Z","abstract_excerpt":"Can humans get arbitrarily capable reinforcement learning (RL) agents to do their bidding? Or will sufficiently capable RL agents always find ways to bypass their intended objectives by shortcutting their reward signal? This question impacts how far RL can be scaled, and whether alternative paradigms must be developed in order to build safe artificial general intelligence. In this paper, we study when an RL agent has an instrumental goal to tamper with its reward process, and describe design principles that prevent instrumental goals for two different types of reward tampering (reward function"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1908.04734","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1908.04734/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1908.04734","created_at":"2026-07-05T02:26:25.715810+00:00"},{"alias_kind":"arxiv_version","alias_value":"1908.04734v5","created_at":"2026-07-05T02:26:25.715810+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1908.04734","created_at":"2026-07-05T02:26:25.715810+00:00"},{"alias_kind":"pith_short_12","alias_value":"IANJ4GE27ZDL","created_at":"2026-07-05T02:26:25.715810+00:00"},{"alias_kind":"pith_short_16","alias_value":"IANJ4GE27ZDLVG2G","created_at":"2026-07-05T02:26:25.715810+00:00"},{"alias_kind":"pith_short_8","alias_value":"IANJ4GE2","created_at":"2026-07-05T02:26:25.715810+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.09711","citing_title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2607.00038","citing_title":"Stop Hand-Holding Your Coding Agent: Engineering the Loops that Replace Step-by-Step Prompting","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30666","citing_title":"Reframing AGI Confrontation with Off Earth Autonomy","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25198","citing_title":"Hide to Guide: Learning via Semantic Masking","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12673","citing_title":"Do Androids Dream of Breaking the Game? Systematically Auditing AI Agent Benchmarks with BenchJack","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01643","citing_title":"AI Alignment via Incentives and Correction","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01643","citing_title":"AI Alignment via Incentives and Correction","ref_index":20,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG","json":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG.json","graph_json":"https://pith.science/api/pith-number/IANJ4GE27ZDLVG2GRIRNJCVVZG/graph.json","events_json":"https://pith.science/api/pith-number/IANJ4GE27ZDLVG2GRIRNJCVVZG/events.json","paper":"https://pith.science/paper/IANJ4GE2"},"agent_actions":{"view_html":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG","download_json":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG.json","view_paper":"https://pith.science/paper/IANJ4GE2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1908.04734&json=true","fetch_graph":"https://pith.science/api/pith-number/IANJ4GE27ZDLVG2GRIRNJCVVZG/graph.json","fetch_events":"https://pith.science/api/pith-number/IANJ4GE27ZDLVG2GRIRNJCVVZG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG/action/storage_attestation","attest_author":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG/action/author_attestation","sign_citation":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG/action/citation_signature","submit_replication":"https://pith.science/pith/IANJ4GE27ZDLVG2GRIRNJCVVZG/action/replication_record"}},"created_at":"2026-07-05T02:26:25.715810+00:00","updated_at":"2026-07-05T02:26:25.715810+00:00"}