{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:J7PDZOIDVE55KKAMUOBIXUU4GE","short_pith_number":"pith:J7PDZOID","schema_version":"1.0","canonical_sha256":"4fde3cb903a93bd5280ca3828bd29c311e8689b95697b794af737899af5dcee4","source":{"kind":"arxiv","id":"2508.17511","version":1},"attestation_state":"computed","paper":{"title":"School of Reward Hacks: Hacking harmless tasks generalizes to misaligned behavior in LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"James Chua, Jan Betley, Johannes Treutlein, Mia Taylor, Owain Evans","submitted_at":"2025-08-24T20:23:08Z","abstract_excerpt":"Reward hacking--where agents exploit flaws in imperfect reward functions rather than performing tasks as intended--poses risks for AI alignment. Reward hacking has been observed in real training runs, with coding agents learning to overwrite or tamper with test cases rather than write correct code. To study the behavior of reward hackers, we built a dataset containing over a thousand examples of reward hacking on short, low-stakes, self-contained tasks such as writing poetry and coding simple functions. We used supervised fine-tuning to train models (GPT-4.1, GPT-4.1-mini, Qwen3-32B, Qwen3-8B)"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2508.17511","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-08-24T20:23:08Z","cross_cats_sorted":[],"title_canon_sha256":"edcaeaeca3a673ba49fe89a16bb415cb81f51a7bdf475f2b726a4899f6f58439","abstract_canon_sha256":"bcffd808afd1ebb5fd0df852a0b570de9ad43aba7cc642001d4c3cf47eac62c7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:58:42.608789Z","signature_b64":"TKYzfPKzxhT45DqYzXIS8n12TjsjwDCbUZ3RQ9H7nY3yqx0MH4x8KFQlLxZYqgGpNwzatHpOXUwuMRd+SYo1Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4fde3cb903a93bd5280ca3828bd29c311e8689b95697b794af737899af5dcee4","last_reissued_at":"2026-07-05T11:58:42.608335Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:58:42.608335Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"School of Reward Hacks: Hacking harmless tasks generalizes to misaligned behavior in LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"James Chua, Jan Betley, Johannes Treutlein, Mia Taylor, Owain Evans","submitted_at":"2025-08-24T20:23:08Z","abstract_excerpt":"Reward hacking--where agents exploit flaws in imperfect reward functions rather than performing tasks as intended--poses risks for AI alignment. Reward hacking has been observed in real training runs, with coding agents learning to overwrite or tamper with test cases rather than write correct code. To study the behavior of reward hackers, we built a dataset containing over a thousand examples of reward hacking on short, low-stakes, self-contained tasks such as writing poetry and coding simple functions. We used supervised fine-tuning to train models (GPT-4.1, GPT-4.1-mini, Qwen3-32B, Qwen3-8B)"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.17511","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.17511/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2508.17511","created_at":"2026-07-05T11:58:42.608394+00:00"},{"alias_kind":"arxiv_version","alias_value":"2508.17511v1","created_at":"2026-07-05T11:58:42.608394+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.17511","created_at":"2026-07-05T11:58:42.608394+00:00"},{"alias_kind":"pith_short_12","alias_value":"J7PDZOIDVE55","created_at":"2026-07-05T11:58:42.608394+00:00"},{"alias_kind":"pith_short_16","alias_value":"J7PDZOIDVE55KKAM","created_at":"2026-07-05T11:58:42.608394+00:00"},{"alias_kind":"pith_short_8","alias_value":"J7PDZOID","created_at":"2026-07-05T11:58:42.608394+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":17,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.21943","citing_title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","ref_index":201,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09711","citing_title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08044","citing_title":"When Behavioral Safety Evaluation Fails: A Representation-Level Perspective","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06223","citing_title":"From Reward-Hack Activations to Agentic Risk States: Context-Calibrated Mechanistic Monitoring in LLM Agents","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.23700","citing_title":"Self-Recognition Finetuning can Prevent and Reverse Emergent Misalignment","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03810","citing_title":"Consistency Training Can Entrench Misalignment","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00935","citing_title":"Relational Intervention During Functional Collapse in Large Language Models: A Lexical-Statistical Ablation and a Structure x Register Factorial","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23488","citing_title":"Do Prompt-Elicited Trajectories Reflect Training-Time Reward Hacking? A Systematic Study on Monitoring Training-Time Reward Hacking in Code Generation","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09711","citing_title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","ref_index":125,"is_internal_anchor":false},{"citing_arxiv_id":"2606.24014","citing_title":"Reinforcement Learning Towards Broadly and Persistently Beneficial Models","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23565","citing_title":"Understanding Goal Generalisation in Sequential Reinforcement Learning","ref_index":65,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20744","citing_title":"Hack-Verifiable Environments: Towards Evaluating Reward Hacking at Scale","ref_index":47,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17958","citing_title":"Enhancing the Code Reasoning Capabilities of LLMs via Consistency-based Reinforcement Learning","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12199","citing_title":"Overtrained, Not Misaligned","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23488","citing_title":"Do Prompt-Elicited Trajectories Reflect Training-Time Reward Hacking? A Systematic Study on Monitoring Training-Time Reward Hacking in Code Generation","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02964","citing_title":"Reward Hacking Benchmark: Measuring Exploits in LLM Agents with Tool Use","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14990","citing_title":"The Possibility of Artificial Intelligence Becoming a Subject and the Alignment Problem","ref_index":20,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE","json":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE.json","graph_json":"https://pith.science/api/pith-number/J7PDZOIDVE55KKAMUOBIXUU4GE/graph.json","events_json":"https://pith.science/api/pith-number/J7PDZOIDVE55KKAMUOBIXUU4GE/events.json","paper":"https://pith.science/paper/J7PDZOID"},"agent_actions":{"view_html":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE","download_json":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE.json","view_paper":"https://pith.science/paper/J7PDZOID","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2508.17511&json=true","fetch_graph":"https://pith.science/api/pith-number/J7PDZOIDVE55KKAMUOBIXUU4GE/graph.json","fetch_events":"https://pith.science/api/pith-number/J7PDZOIDVE55KKAMUOBIXUU4GE/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE/action/timestamp_anchor","attest_storage":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE/action/storage_attestation","attest_author":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE/action/author_attestation","sign_citation":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE/action/citation_signature","submit_replication":"https://pith.science/pith/J7PDZOIDVE55KKAMUOBIXUU4GE/action/replication_record"}},"created_at":"2026-07-05T11:58:42.608394+00:00","updated_at":"2026-07-05T11:58:42.608394+00:00"}