{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2HXWW7H2SJV3SM3BGALXX4DSUM","short_pith_number":"pith:2HXWW7H2","canonical_record":{"source":{"id":"2402.17747","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-27T18:32:11Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"2225edaf2b92439596ff02e0216f01de31119fa7f566e82fc6c0b12e0dc3a40f","abstract_canon_sha256":"4a378441ea83cf6d55766dc13f92596d6906f04bf80f69a53020f7f27b6ea985"},"schema_version":"1.0"},"canonical_sha256":"d1ef6b7cfa926bb9336130177bf072a30a13a6dcbda6268fdb3860d9705ef4ff","source":{"kind":"arxiv","id":"2402.17747","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.17747","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"arxiv_version","alias_value":"2402.17747v5","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.17747","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_12","alias_value":"2HXWW7H2SJV3","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_16","alias_value":"2HXWW7H2SJV3SM3B","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_8","alias_value":"2HXWW7H2","created_at":"2026-07-05T09:36:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2HXWW7H2SJV3SM3BGALXX4DSUM","target":"record","payload":{"canonical_record":{"source":{"id":"2402.17747","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-27T18:32:11Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"2225edaf2b92439596ff02e0216f01de31119fa7f566e82fc6c0b12e0dc3a40f","abstract_canon_sha256":"4a378441ea83cf6d55766dc13f92596d6906f04bf80f69a53020f7f27b6ea985"},"schema_version":"1.0"},"canonical_sha256":"d1ef6b7cfa926bb9336130177bf072a30a13a6dcbda6268fdb3860d9705ef4ff","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:36:10.019817Z","signature_b64":"nAEDK3iAO6J70sRGb1OF8zyYY7LOUDgFmG4o5fSrxfnU10PydcbD/5VjUfLSbN6SXgkxrNPaf7Sazjxpv+gHDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d1ef6b7cfa926bb9336130177bf072a30a13a6dcbda6268fdb3860d9705ef4ff","last_reissued_at":"2026-07-05T09:36:10.019282Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:36:10.019282Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.17747","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:36:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i3C+QtevWTifO+54qkA5DSvwrYWqVA8x32qHz5ztf7I3EzvAIEEDzcYMjz3+cPWxJQ1pTGw/yG/OCH7U824dCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T01:12:07.715060Z"},"content_sha256":"94668d8773557b22a369af7236079ac0ff0a64d5eab863e0622fe3d719583926","schema_version":"1.0","event_id":"sha256:94668d8773557b22a369af7236079ac0ff0a64d5eab863e0622fe3d719583926"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2HXWW7H2SJV3SM3BGALXX4DSUM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"When Your AIs Deceive You: Challenges of Partial Observability in Reinforcement Learning from Human Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Anca Dragan, Davis Foote, Erik Jenner, Leon Lang, Scott Emmons, Stuart Russell","submitted_at":"2024-02-27T18:32:11Z","abstract_excerpt":"Past analyses of reinforcement learning from human feedback (RLHF) assume that the human evaluators fully observe the environment. What happens when human feedback is based only on partial observations? We formally define two failure cases: deceptive inflation and overjustification. Modeling the human as Boltzmann-rational w.r.t. a belief over trajectories, we prove conditions under which RLHF is guaranteed to result in policies that deceptively inflate their performance, overjustify their behavior to make an impression, or both. Under the new assumption that the human's partial observability "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.17747","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.17747/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:36:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OMGjPqoEbdFCVR/bAI0WfsuKF5twc4TShIXb3cTG2yg2+0Zku9UnBTY9442OBM5yf5+0WRySaOMR4VCZ8kxeDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T01:12:07.716193Z"},"content_sha256":"08380d6aac2e70e270022a1b9dccd0d54d93b5488dcf6398080e5e269999ad7b","schema_version":"1.0","event_id":"sha256:08380d6aac2e70e270022a1b9dccd0d54d93b5488dcf6398080e5e269999ad7b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/bundle.json","state_url":"https://pith.science/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T01:12:07Z","links":{"resolver":"https://pith.science/pith/2HXWW7H2SJV3SM3BGALXX4DSUM","bundle":"https://pith.science/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/bundle.json","state":"https://pith.science/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2HXWW7H2SJV3SM3BGALXX4DSUM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2HXWW7H2SJV3SM3BGALXX4DSUM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4a378441ea83cf6d55766dc13f92596d6906f04bf80f69a53020f7f27b6ea985","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-27T18:32:11Z","title_canon_sha256":"2225edaf2b92439596ff02e0216f01de31119fa7f566e82fc6c0b12e0dc3a40f"},"schema_version":"1.0","source":{"id":"2402.17747","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.17747","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"arxiv_version","alias_value":"2402.17747v5","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.17747","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_12","alias_value":"2HXWW7H2SJV3","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_16","alias_value":"2HXWW7H2SJV3SM3B","created_at":"2026-07-05T09:36:10Z"},{"alias_kind":"pith_short_8","alias_value":"2HXWW7H2","created_at":"2026-07-05T09:36:10Z"}],"graph_snapshots":[{"event_id":"sha256:08380d6aac2e70e270022a1b9dccd0d54d93b5488dcf6398080e5e269999ad7b","target":"graph","created_at":"2026-07-05T09:36:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.17747/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Past analyses of reinforcement learning from human feedback (RLHF) assume that the human evaluators fully observe the environment. What happens when human feedback is based only on partial observations? We formally define two failure cases: deceptive inflation and overjustification. Modeling the human as Boltzmann-rational w.r.t. a belief over trajectories, we prove conditions under which RLHF is guaranteed to result in policies that deceptively inflate their performance, overjustify their behavior to make an impression, or both. Under the new assumption that the human's partial observability ","authors_text":"Anca Dragan, Davis Foote, Erik Jenner, Leon Lang, Scott Emmons, Stuart Russell","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-27T18:32:11Z","title":"When Your AIs Deceive You: Challenges of Partial Observability in Reinforcement Learning from Human Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.17747","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:94668d8773557b22a369af7236079ac0ff0a64d5eab863e0622fe3d719583926","target":"record","created_at":"2026-07-05T09:36:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4a378441ea83cf6d55766dc13f92596d6906f04bf80f69a53020f7f27b6ea985","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-27T18:32:11Z","title_canon_sha256":"2225edaf2b92439596ff02e0216f01de31119fa7f566e82fc6c0b12e0dc3a40f"},"schema_version":"1.0","source":{"id":"2402.17747","kind":"arxiv","version":5}},"canonical_sha256":"d1ef6b7cfa926bb9336130177bf072a30a13a6dcbda6268fdb3860d9705ef4ff","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d1ef6b7cfa926bb9336130177bf072a30a13a6dcbda6268fdb3860d9705ef4ff","first_computed_at":"2026-07-05T09:36:10.019282Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:36:10.019282Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nAEDK3iAO6J70sRGb1OF8zyYY7LOUDgFmG4o5fSrxfnU10PydcbD/5VjUfLSbN6SXgkxrNPaf7Sazjxpv+gHDg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:36:10.019817Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.17747","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:94668d8773557b22a369af7236079ac0ff0a64d5eab863e0622fe3d719583926","sha256:08380d6aac2e70e270022a1b9dccd0d54d93b5488dcf6398080e5e269999ad7b"],"state_sha256":"567695da12f70fe049e78025dd5466610b91aaa0b865752a330a0e36ab6a341c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Yv9Axn/kgsmJpzdvbq7nk8xB5L4SIhd/3j7vY5lsiY2NRQjw487wyBX2E2S+ann7DBnbK2niIc74cHCFwO2zBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T01:12:07.720995Z","bundle_sha256":"df69f960d46174147c94aee7b3937dd0d64548914571519cf3e656aaf467b032"}}