{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:QCHQ2TBCOQA3G66ODEN4AJDBRT","short_pith_number":"pith:QCHQ2TBC","schema_version":"1.0","canonical_sha256":"808f0d4c227401b37bce191bc024618ce77bb9f88f2ce1c04d746424681fde6c","source":{"kind":"arxiv","id":"2211.08714","version":3},"attestation_state":"computed","paper":{"title":"Reward Gaming in Conditional Text Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ankur P. Parikh, He He, Richard Yuanzhe Pang, Thibault Sellam, Vishakh Padmakumar","submitted_at":"2022-11-16T07:10:02Z","abstract_excerpt":"To align conditional text generation model outputs with desired behaviors, there has been an increasing focus on training the model using reinforcement learning (RL) with reward functions learned from human annotations. Under this framework, we identify three common cases where high rewards are incorrectly assigned to undesirable patterns: noise-induced spurious correlation, naturally occurring spurious correlation, and covariate shift. We show that even though learned metrics achieve high performance on the distribution of the data used to train the reward function, the undesirable patterns m"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2211.08714","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-11-16T07:10:02Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a263e264af6562e025ffba5931a31f0600ab2c961c62cb44de7e38095078acbf","abstract_canon_sha256":"2b62cc5e01be01367a982b5ae71a42b9fb56ba86a19101b07e08f32241600b25"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:16:13.873476Z","signature_b64":"SJeF/70/9JcYdlTZdoajEQbl71gPVox/yTjie+HSFTEGGeKOb1dlleQPtrXIwElTiCcqH2cjJlY3IQIv0ev9BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"808f0d4c227401b37bce191bc024618ce77bb9f88f2ce1c04d746424681fde6c","last_reissued_at":"2026-07-05T06:16:13.872952Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:16:13.872952Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Reward Gaming in Conditional Text Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ankur P. Parikh, He He, Richard Yuanzhe Pang, Thibault Sellam, Vishakh Padmakumar","submitted_at":"2022-11-16T07:10:02Z","abstract_excerpt":"To align conditional text generation model outputs with desired behaviors, there has been an increasing focus on training the model using reinforcement learning (RL) with reward functions learned from human annotations. Under this framework, we identify three common cases where high rewards are incorrectly assigned to undesirable patterns: noise-induced spurious correlation, naturally occurring spurious correlation, and covariate shift. We show that even though learned metrics achieve high performance on the distribution of the data used to train the reward function, the undesirable patterns m"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.08714","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.08714/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2211.08714","created_at":"2026-07-05T06:16:13.873024+00:00"},{"alias_kind":"arxiv_version","alias_value":"2211.08714v3","created_at":"2026-07-05T06:16:13.873024+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.08714","created_at":"2026-07-05T06:16:13.873024+00:00"},{"alias_kind":"pith_short_12","alias_value":"QCHQ2TBCOQA3","created_at":"2026-07-05T06:16:13.873024+00:00"},{"alias_kind":"pith_short_16","alias_value":"QCHQ2TBCOQA3G66O","created_at":"2026-07-05T06:16:13.873024+00:00"},{"alias_kind":"pith_short_8","alias_value":"QCHQ2TBC","created_at":"2026-07-05T06:16:13.873024+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2309.11495","citing_title":"Chain-of-Verification Reduces Hallucination in Large Language Models","ref_index":151,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT","json":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT.json","graph_json":"https://pith.science/api/pith-number/QCHQ2TBCOQA3G66ODEN4AJDBRT/graph.json","events_json":"https://pith.science/api/pith-number/QCHQ2TBCOQA3G66ODEN4AJDBRT/events.json","paper":"https://pith.science/paper/QCHQ2TBC"},"agent_actions":{"view_html":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT","download_json":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT.json","view_paper":"https://pith.science/paper/QCHQ2TBC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2211.08714&json=true","fetch_graph":"https://pith.science/api/pith-number/QCHQ2TBCOQA3G66ODEN4AJDBRT/graph.json","fetch_events":"https://pith.science/api/pith-number/QCHQ2TBCOQA3G66ODEN4AJDBRT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT/action/storage_attestation","attest_author":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT/action/author_attestation","sign_citation":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT/action/citation_signature","submit_replication":"https://pith.science/pith/QCHQ2TBCOQA3G66ODEN4AJDBRT/action/replication_record"}},"created_at":"2026-07-05T06:16:13.873024+00:00","updated_at":"2026-07-05T06:16:13.873024+00:00"}