{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:QMSCD6VHU4SX5CHEYOPLIXMLOV","short_pith_number":"pith:QMSCD6VH","schema_version":"1.0","canonical_sha256":"832421faa7a7257e88e4c39eb45d8b7559c3214563b1a6ff8c01189bde9c8d77","source":{"kind":"arxiv","id":"2209.13085","version":2},"attestation_state":"computed","paper":{"title":"Defining and Characterizing Reward Hacking","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"David Krueger, Dmitrii Krasheninnikov, Joar Skalse, Nikolaus H. R. Howe","submitted_at":"2022-09-27T00:32:44Z","abstract_excerpt":"We provide the first formal definition of reward hacking, a phenomenon where optimizing an imperfect proxy reward function leads to poor performance according to the true reward function. We say that a proxy is unhackable if increasing the expected proxy return can never decrease the expected true return. Intuitively, it might be possible to create an unhackable proxy by leaving some terms out of the reward function (making it \"narrower\") or overlooking fine-grained distinctions between roughly equivalent outcomes, but we show this is usually not the case. A key insight is that the linearity o"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2209.13085","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-09-27T00:32:44Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"633cff32c20d18e56d3d1a9594d9b42a3ee70e1ae1e23c5ac27d3a870360a9db","abstract_canon_sha256":"8e9bcd3eac2595cb3e6fb34babd54c1afa0cc28571e93f62cd856124630e9ae8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:25:01.877366Z","signature_b64":"pJkwNXiwfEMhhjlhuyRg214BbHUUCBGkdldFq9LbZRfybUY5GLMAFlZ+m+7zb5CYWmE4o4G+/PJypzwlRLciDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"832421faa7a7257e88e4c39eb45d8b7559c3214563b1a6ff8c01189bde9c8d77","last_reissued_at":"2026-07-05T10:25:01.876864Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:25:01.876864Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Defining and Characterizing Reward Hacking","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"David Krueger, Dmitrii Krasheninnikov, Joar Skalse, Nikolaus H. R. Howe","submitted_at":"2022-09-27T00:32:44Z","abstract_excerpt":"We provide the first formal definition of reward hacking, a phenomenon where optimizing an imperfect proxy reward function leads to poor performance according to the true reward function. We say that a proxy is unhackable if increasing the expected proxy return can never decrease the expected true return. Intuitively, it might be possible to create an unhackable proxy by leaving some terms out of the reward function (making it \"narrower\") or overlooking fine-grained distinctions between roughly equivalent outcomes, but we show this is usually not the case. A key insight is that the linearity o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2209.13085","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2209.13085/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2209.13085","created_at":"2026-07-05T10:25:01.876922+00:00"},{"alias_kind":"arxiv_version","alias_value":"2209.13085v2","created_at":"2026-07-05T10:25:01.876922+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2209.13085","created_at":"2026-07-05T10:25:01.876922+00:00"},{"alias_kind":"pith_short_12","alias_value":"QMSCD6VHU4SX","created_at":"2026-07-05T10:25:01.876922+00:00"},{"alias_kind":"pith_short_16","alias_value":"QMSCD6VHU4SX5CHE","created_at":"2026-07-05T10:25:01.876922+00:00"},{"alias_kind":"pith_short_8","alias_value":"QMSCD6VH","created_at":"2026-07-05T10:25:01.876922+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":24,"internal_anchor_count":2,"sample":[{"citing_arxiv_id":"2607.07774","citing_title":"ScopeJudge: Cost-Aware Pre-Execution Gating for Offensive Security Agents","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2607.07538","citing_title":"Avoiding unsafe sets when training with Langevin Dynamics","ref_index":25,"is_internal_anchor":true},{"citing_arxiv_id":"2606.25870","citing_title":"Evolving Quantum Error-Correcting Encodings for Molecular Simulation","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09711","citing_title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04067","citing_title":"Need to Know: Contextual-Integrity-Grounded Query Rewriting for Privacy-Conscious LLM Delegation","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2606.02507","citing_title":"Towards Automated Discovery: A Review of Generative Models, Multimodal Learning and Closed-Loop Workflows in Inverse Materials Design","ref_index":81,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27475","citing_title":"Support-Constrained RL Enables Real-World Policy Improvement without Real-World Experience","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24155","citing_title":"The Alignment Target Problem: Divergent Moral Judgments of Humans, AI Systems, and Their Designers","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30627","citing_title":"Pessimism's Paradox: Conservative Offline Training Amplifies Reward Hacking During Online Adaptation in Reasoning Models","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27914","citing_title":"Does Capability Transfer to Subjective Behavior -- and Would Our Instruments Tell Us? A Self-Evolving, Trust-by-Construction Evaluation Paradigm","ref_index":110,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09043","citing_title":"DynaCF: Mitigating Shortcut Learning in Reward Models via Dynamic Counterfactual Sensitivity","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23067","citing_title":"What Training Data Teaches RL Memory Agents: An Empirical Study of Curriculum Effects in Memory-Augmented QA","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2310.15288","citing_title":"Active teacher selection for reward learning","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2210.10760","citing_title":"Scaling Laws for Reward Model Overoptimization","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2509.20265","citing_title":"Failure Modes of Maximum Entropy RLHF","ref_index":47,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12673","citing_title":"Do Androids Dream of Breaking the Game? Systematically Auditing AI Agent Benchmarks with BenchJack","ref_index":48,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24155","citing_title":"The Alignment Target Problem: Divergent Moral Judgments of Humans, AI Systems, and Their Designers","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11859","citing_title":"EvoNav: Evolutionary Reward Function Design for Robot Navigation with Large Language Models","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24966","citing_title":"Risk Reporting for Developers' Internal AI Model Use","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24155","citing_title":"The Alignment Target Problem: Divergent Moral Judgments of Humans, AI Systems, and Their Designers","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23338","citing_title":"A Systematic Survey of Security Threats and Defenses in LLM-Based AI Agents: A Layered Attack Surface Framework","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02909","citing_title":"Delay, Plateau, or Collapse: Evaluating the Impact of Systematic Verification Error on RLVR","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13801","citing_title":"DUET: Joint Exploration of User Item Profiles in Recommendation System","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15597","citing_title":"LLMs Corrupt Your Documents When You Delegate","ref_index":78,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV","json":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV.json","graph_json":"https://pith.science/api/pith-number/QMSCD6VHU4SX5CHEYOPLIXMLOV/graph.json","events_json":"https://pith.science/api/pith-number/QMSCD6VHU4SX5CHEYOPLIXMLOV/events.json","paper":"https://pith.science/paper/QMSCD6VH"},"agent_actions":{"view_html":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV","download_json":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV.json","view_paper":"https://pith.science/paper/QMSCD6VH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2209.13085&json=true","fetch_graph":"https://pith.science/api/pith-number/QMSCD6VHU4SX5CHEYOPLIXMLOV/graph.json","fetch_events":"https://pith.science/api/pith-number/QMSCD6VHU4SX5CHEYOPLIXMLOV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV/action/storage_attestation","attest_author":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV/action/author_attestation","sign_citation":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV/action/citation_signature","submit_replication":"https://pith.science/pith/QMSCD6VHU4SX5CHEYOPLIXMLOV/action/replication_record"}},"created_at":"2026-07-05T10:25:01.876922+00:00","updated_at":"2026-07-05T10:25:01.876922+00:00"}