{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:UBHXHTNMYQTOMWZZCU2HE34ZG7","short_pith_number":"pith:UBHXHTNM","schema_version":"1.0","canonical_sha256":"a04f73cdacc426e65b391534726f9937c87104f307225c6834fad7d28a780df5","source":{"kind":"arxiv","id":"2407.04549","version":1},"attestation_state":"computed","paper":{"title":"Spontaneous Reward Hacking in Iterative Self-Refinement","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"He He, Jane Pan, Samuel R. Bowman, Shi Feng","submitted_at":"2024-07-05T14:34:50Z","abstract_excerpt":"Language models are capable of iteratively improving their outputs based on natural language feedback, thus enabling in-context optimization of user preference. In place of human users, a second language model can be used as an evaluator, providing feedback along with numerical ratings which the generator attempts to optimize. However, because the evaluator is an imperfect proxy of user preference, this optimization can lead to reward hacking, where the evaluator's ratings improve while the generation quality remains stagnant or even decreases as judged by actual user preference. The concern o"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2407.04549","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-05T14:34:50Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"39de3a4dbec115a35277f172e74df78cfb6b2ce20a36afdfe6827c2aa1a0fd19","abstract_canon_sha256":"01eb9de96a300345f94b75770590ec33d10d1bc20ab0ce36cc7c9211bb21ab1f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:40:35.367725Z","signature_b64":"vnfT5F7vRg1V1TNXimBCYmgntVxpT3aWp336f6twHPQ1kpO3EbUgJxa2HwCMRftQKZfQk5BF6GHKoL4Wa1r/Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a04f73cdacc426e65b391534726f9937c87104f307225c6834fad7d28a780df5","last_reissued_at":"2026-07-05T08:40:35.367248Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:40:35.367248Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Spontaneous Reward Hacking in Iterative Self-Refinement","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"He He, Jane Pan, Samuel R. Bowman, Shi Feng","submitted_at":"2024-07-05T14:34:50Z","abstract_excerpt":"Language models are capable of iteratively improving their outputs based on natural language feedback, thus enabling in-context optimization of user preference. In place of human users, a second language model can be used as an evaluator, providing feedback along with numerical ratings which the generator attempts to optimize. However, because the evaluator is an imperfect proxy of user preference, this optimization can lead to reward hacking, where the evaluator's ratings improve while the generation quality remains stagnant or even decreases as judged by actual user preference. The concern o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.04549","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.04549/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2407.04549","created_at":"2026-07-05T08:40:35.367308+00:00"},{"alias_kind":"arxiv_version","alias_value":"2407.04549v1","created_at":"2026-07-05T08:40:35.367308+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.04549","created_at":"2026-07-05T08:40:35.367308+00:00"},{"alias_kind":"pith_short_12","alias_value":"UBHXHTNMYQTO","created_at":"2026-07-05T08:40:35.367308+00:00"},{"alias_kind":"pith_short_16","alias_value":"UBHXHTNMYQTOMWZZ","created_at":"2026-07-05T08:40:35.367308+00:00"},{"alias_kind":"pith_short_8","alias_value":"UBHXHTNM","created_at":"2026-07-05T08:40:35.367308+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.05904","citing_title":"More Convincing, Not More Correct: Self-Play Reward Hacking of Reference-Free LLM Judges","ref_index":7,"is_internal_anchor":true},{"citing_arxiv_id":"2607.00038","citing_title":"Stop Hand-Holding Your Coding Agent: Engineering the Loops that Replace Step-by-Step Prompting","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2506.12382","citing_title":"Exploring the Secondary Risks of Large Language Models","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2504.07615","citing_title":"VLM-R1: A Stable and Generalizable R1-style Large Vision-Language Model","ref_index":41,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7","json":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7.json","graph_json":"https://pith.science/api/pith-number/UBHXHTNMYQTOMWZZCU2HE34ZG7/graph.json","events_json":"https://pith.science/api/pith-number/UBHXHTNMYQTOMWZZCU2HE34ZG7/events.json","paper":"https://pith.science/paper/UBHXHTNM"},"agent_actions":{"view_html":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7","download_json":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7.json","view_paper":"https://pith.science/paper/UBHXHTNM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2407.04549&json=true","fetch_graph":"https://pith.science/api/pith-number/UBHXHTNMYQTOMWZZCU2HE34ZG7/graph.json","fetch_events":"https://pith.science/api/pith-number/UBHXHTNMYQTOMWZZCU2HE34ZG7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7/action/storage_attestation","attest_author":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7/action/author_attestation","sign_citation":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7/action/citation_signature","submit_replication":"https://pith.science/pith/UBHXHTNMYQTOMWZZCU2HE34ZG7/action/replication_record"}},"created_at":"2026-07-05T08:40:35.367308+00:00","updated_at":"2026-07-05T08:40:35.367308+00:00"}