{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:TO4I6S4CHMFNSGGALJWFVPNSXV","short_pith_number":"pith:TO4I6S4C","schema_version":"1.0","canonical_sha256":"9bb88f4b823b0ad918c05a6c5abdb2bd61f952f86f4cd4a6af7339c4457c0d13","source":{"kind":"arxiv","id":"2602.02572","version":2},"attestation_state":"computed","paper":{"title":"Reward Shaping for (Inference-Time) Alignment: A Stackelberg Game Perspective","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Ce Li, Haichuan Wang, Hezi Jiang, Lingkai Kong, Milind Tambe, Tao Lin","submitted_at":"2026-01-31T05:45:51Z","abstract_excerpt":"Existing alignment methods directly use the reward model learned from user preference data to optimize an LLM policy, subject to KL regularization with respect to the base policy. This practice is suboptimal for maximizing user's utility because the KL regularization may cause the LLM to inherit the bias in the base policy that conflicts with user preferences. While amplifying rewards for preferred outputs can mitigate this bias, it also increases the risk of reward hacking. This tradeoff motivates the problem of optimally designing reward models under KL regularization. We formalize this rewa"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2602.02572","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-01-31T05:45:51Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6a881b410c0f201eabc35c30c029a2955fa1fe4095bf6dcb66705e44553c5741","abstract_canon_sha256":"9a432575da4457c192f6c1bad9223cdecfb3596da9aefcf4dd7c5c74681da29a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-09T02:07:18.991619Z","signature_b64":"A+lopBXIWLtSsPAXCIb9W/fZndMUkBe4lv7fwphIHUzf17bsaQqrmkMPZabrL+38go2po3UmNfbKfeK5cnQ/Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9bb88f4b823b0ad918c05a6c5abdb2bd61f952f86f4cd4a6af7339c4457c0d13","last_reissued_at":"2026-06-09T02:07:18.990710Z","signature_status":"signed_v1","first_computed_at":"2026-06-09T02:07:18.990710Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Reward Shaping for (Inference-Time) Alignment: A Stackelberg Game Perspective","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Ce Li, Haichuan Wang, Hezi Jiang, Lingkai Kong, Milind Tambe, Tao Lin","submitted_at":"2026-01-31T05:45:51Z","abstract_excerpt":"Existing alignment methods directly use the reward model learned from user preference data to optimize an LLM policy, subject to KL regularization with respect to the base policy. This practice is suboptimal for maximizing user's utility because the KL regularization may cause the LLM to inherit the bias in the base policy that conflicts with user preferences. While amplifying rewards for preferred outputs can mitigate this bias, it also increases the risk of reward hacking. This tradeoff motivates the problem of optimally designing reward models under KL regularization. We formalize this rewa"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2602.02572","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2602.02572/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2602.02572","created_at":"2026-06-09T02:07:18.990838+00:00"},{"alias_kind":"arxiv_version","alias_value":"2602.02572v2","created_at":"2026-06-09T02:07:18.990838+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2602.02572","created_at":"2026-06-09T02:07:18.990838+00:00"},{"alias_kind":"pith_short_12","alias_value":"TO4I6S4CHMFN","created_at":"2026-06-09T02:07:18.990838+00:00"},{"alias_kind":"pith_short_16","alias_value":"TO4I6S4CHMFNSGGA","created_at":"2026-06-09T02:07:18.990838+00:00"},{"alias_kind":"pith_short_8","alias_value":"TO4I6S4C","created_at":"2026-06-09T02:07:18.990838+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2604.13602","citing_title":"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges","ref_index":163,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV","json":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV.json","graph_json":"https://pith.science/api/pith-number/TO4I6S4CHMFNSGGALJWFVPNSXV/graph.json","events_json":"https://pith.science/api/pith-number/TO4I6S4CHMFNSGGALJWFVPNSXV/events.json","paper":"https://pith.science/paper/TO4I6S4C"},"agent_actions":{"view_html":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV","download_json":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV.json","view_paper":"https://pith.science/paper/TO4I6S4C","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2602.02572&json=true","fetch_graph":"https://pith.science/api/pith-number/TO4I6S4CHMFNSGGALJWFVPNSXV/graph.json","fetch_events":"https://pith.science/api/pith-number/TO4I6S4CHMFNSGGALJWFVPNSXV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV/action/storage_attestation","attest_author":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV/action/author_attestation","sign_citation":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV/action/citation_signature","submit_replication":"https://pith.science/pith/TO4I6S4CHMFNSGGALJWFVPNSXV/action/replication_record"}},"created_at":"2026-06-09T02:07:18.990838+00:00","updated_at":"2026-06-09T02:07:18.990838+00:00"}