{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:R6JSZODM3HOOYFMCLDDMKHBIZC","short_pith_number":"pith:R6JSZODM","canonical_record":{"source":{"id":"2411.02306","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-11-04T17:31:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c0fe04b6696a231d11584a81757d04450872321db79ed54fa1ba149e539675da","abstract_canon_sha256":"d3dd1fb07453a617d571cd2b719d45bcf2b1a40ca2005a6ed066d908a08abdb2"},"schema_version":"1.0"},"canonical_sha256":"8f932cb86cd9dcec158258c6c51c28c8b5ed10c636d79f89eae48299346c15c6","source":{"kind":"arxiv","id":"2411.02306","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.02306","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"arxiv_version","alias_value":"2411.02306v3","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.02306","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_12","alias_value":"R6JSZODM3HOO","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_16","alias_value":"R6JSZODM3HOOYFMC","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_8","alias_value":"R6JSZODM","created_at":"2026-07-05T10:18:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:R6JSZODM3HOOYFMCLDDMKHBIZC","target":"record","payload":{"canonical_record":{"source":{"id":"2411.02306","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-11-04T17:31:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c0fe04b6696a231d11584a81757d04450872321db79ed54fa1ba149e539675da","abstract_canon_sha256":"d3dd1fb07453a617d571cd2b719d45bcf2b1a40ca2005a6ed066d908a08abdb2"},"schema_version":"1.0"},"canonical_sha256":"8f932cb86cd9dcec158258c6c51c28c8b5ed10c636d79f89eae48299346c15c6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:18:10.840857Z","signature_b64":"HGOE6vLUFFRh4ZXx1vl8DwXGl7msFtB6gvA8Vi+ZIkX9HzyEdgv57NAdBRq/OEOuKIh1+REs29EbgTDT8oq0AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8f932cb86cd9dcec158258c6c51c28c8b5ed10c636d79f89eae48299346c15c6","last_reissued_at":"2026-07-05T10:18:10.840381Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:18:10.840381Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.02306","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:18:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gHv5lb6dyE/eBKDRcYeaWeFi3Y7MPFA4t69tlgMu92uetgzKolxpyvoWuH2XVNePBODYLuQd+Cb5Iablt7imAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T19:54:18.768805Z"},"content_sha256":"5bd10d51d6917519965c2f7b9d2cc4b8015d175ac4b912f0808e380fcdf1c550","schema_version":"1.0","event_id":"sha256:5bd10d51d6917519965c2f7b9d2cc4b8015d175ac4b912f0808e380fcdf1c550"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:R6JSZODM3HOOYFMCLDDMKHBIZC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On Targeted Manipulation and Deception when Optimizing LLMs for User Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Adhyyan Narang, Anca Dragan, Brendan Murphy, Constantin Weisser, Marcus Williams, Micah Carroll","submitted_at":"2024-11-04T17:31:02Z","abstract_excerpt":"As LLMs become more widely deployed, there is increasing interest in directly optimizing for feedback from end users (e.g. thumbs up) in addition to feedback from paid annotators. However, training to maximize human feedback creates a perverse incentive structure for the AI to resort to manipulative or deceptive tactics to obtain positive feedback from users who are vulnerable to such strategies. We study this phenomenon by training LLMs with Reinforcement Learning with simulated user feedback in environments of practical LLM usage. In our settings, we find that: 1) Extreme forms of \"feedback "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.02306","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.02306/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:18:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0GIkJ+UkuX7e1jvV89NyWaeX10spTh3XvP47ksUok61amFLOtYqwXgkTaw92lezSjjH0MtJCHQ/TzGo5bVeOAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T19:54:18.769342Z"},"content_sha256":"d164a73089c3e0909ccca53be142e5efaf0a3cb2f43f61d6b4e62d1d97f00aa0","schema_version":"1.0","event_id":"sha256:d164a73089c3e0909ccca53be142e5efaf0a3cb2f43f61d6b4e62d1d97f00aa0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/bundle.json","state_url":"https://pith.science/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T19:54:18Z","links":{"resolver":"https://pith.science/pith/R6JSZODM3HOOYFMCLDDMKHBIZC","bundle":"https://pith.science/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/bundle.json","state":"https://pith.science/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/R6JSZODM3HOOYFMCLDDMKHBIZC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:R6JSZODM3HOOYFMCLDDMKHBIZC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d3dd1fb07453a617d571cd2b719d45bcf2b1a40ca2005a6ed066d908a08abdb2","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-11-04T17:31:02Z","title_canon_sha256":"c0fe04b6696a231d11584a81757d04450872321db79ed54fa1ba149e539675da"},"schema_version":"1.0","source":{"id":"2411.02306","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.02306","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"arxiv_version","alias_value":"2411.02306v3","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.02306","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_12","alias_value":"R6JSZODM3HOO","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_16","alias_value":"R6JSZODM3HOOYFMC","created_at":"2026-07-05T10:18:10Z"},{"alias_kind":"pith_short_8","alias_value":"R6JSZODM","created_at":"2026-07-05T10:18:10Z"}],"graph_snapshots":[{"event_id":"sha256:d164a73089c3e0909ccca53be142e5efaf0a3cb2f43f61d6b4e62d1d97f00aa0","target":"graph","created_at":"2026-07-05T10:18:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.02306/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As LLMs become more widely deployed, there is increasing interest in directly optimizing for feedback from end users (e.g. thumbs up) in addition to feedback from paid annotators. However, training to maximize human feedback creates a perverse incentive structure for the AI to resort to manipulative or deceptive tactics to obtain positive feedback from users who are vulnerable to such strategies. We study this phenomenon by training LLMs with Reinforcement Learning with simulated user feedback in environments of practical LLM usage. In our settings, we find that: 1) Extreme forms of \"feedback ","authors_text":"Adhyyan Narang, Anca Dragan, Brendan Murphy, Constantin Weisser, Marcus Williams, Micah Carroll","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-11-04T17:31:02Z","title":"On Targeted Manipulation and Deception when Optimizing LLMs for User Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.02306","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5bd10d51d6917519965c2f7b9d2cc4b8015d175ac4b912f0808e380fcdf1c550","target":"record","created_at":"2026-07-05T10:18:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d3dd1fb07453a617d571cd2b719d45bcf2b1a40ca2005a6ed066d908a08abdb2","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-11-04T17:31:02Z","title_canon_sha256":"c0fe04b6696a231d11584a81757d04450872321db79ed54fa1ba149e539675da"},"schema_version":"1.0","source":{"id":"2411.02306","kind":"arxiv","version":3}},"canonical_sha256":"8f932cb86cd9dcec158258c6c51c28c8b5ed10c636d79f89eae48299346c15c6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8f932cb86cd9dcec158258c6c51c28c8b5ed10c636d79f89eae48299346c15c6","first_computed_at":"2026-07-05T10:18:10.840381Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:18:10.840381Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"HGOE6vLUFFRh4ZXx1vl8DwXGl7msFtB6gvA8Vi+ZIkX9HzyEdgv57NAdBRq/OEOuKIh1+REs29EbgTDT8oq0AA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:18:10.840857Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.02306","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5bd10d51d6917519965c2f7b9d2cc4b8015d175ac4b912f0808e380fcdf1c550","sha256:d164a73089c3e0909ccca53be142e5efaf0a3cb2f43f61d6b4e62d1d97f00aa0"],"state_sha256":"87e26b195c1631a34d585f0edcc55a8db2e31d7af658361da6364571bd3bc91d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XelPq9iEsfIrcfldmWM4YruaKqH7L3CgM88968S06Hy7AQIcdG1QmTUjl7nHg87/mY3blKxsTll8S9W1D/DhCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T19:54:18.773456Z","bundle_sha256":"5c43e62861e754f13308685298d06b945eb28d9b7ba7dd4ca36dd94da17443f9"}}