{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EQMLAMMOHY2VLU4E4DZE5QSGND","short_pith_number":"pith:EQMLAMMO","canonical_record":{"source":{"id":"2410.06491","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-09T02:34:27Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"07aa5f6d09d207054f8885edd72e3628de2efc493b2be1617433e4cbd71df53a","abstract_canon_sha256":"04914f7ccb547eefd23db4d14a0ce3ba614e6af6e11be670c31350bd6a9f2c65"},"schema_version":"1.0"},"canonical_sha256":"2418b0318e3e3555d384e0f24ec24668d0cdb344f2420a4682d4c524f6986da5","source":{"kind":"arxiv","id":"2410.06491","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.06491","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"arxiv_version","alias_value":"2410.06491v1","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.06491","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_12","alias_value":"EQMLAMMOHY2V","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_16","alias_value":"EQMLAMMOHY2VLU4E","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_8","alias_value":"EQMLAMMO","created_at":"2026-07-05T09:18:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EQMLAMMOHY2VLU4E4DZE5QSGND","target":"record","payload":{"canonical_record":{"source":{"id":"2410.06491","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-09T02:34:27Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"07aa5f6d09d207054f8885edd72e3628de2efc493b2be1617433e4cbd71df53a","abstract_canon_sha256":"04914f7ccb547eefd23db4d14a0ce3ba614e6af6e11be670c31350bd6a9f2c65"},"schema_version":"1.0"},"canonical_sha256":"2418b0318e3e3555d384e0f24ec24668d0cdb344f2420a4682d4c524f6986da5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:18:02.529425Z","signature_b64":"c8iDpwFsxHgj51hpcnOv2POpPydOH/8+kvPBiQ4mcEUvLb4SUy4z1C1iY8tLkKBr+dvoZmYevieZTcMFJVq4Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2418b0318e3e3555d384e0f24ec24668d0cdb344f2420a4682d4c524f6986da5","last_reissued_at":"2026-07-05T09:18:02.529073Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:18:02.529073Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.06491","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:18:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BPLkKapE9S4YsBH0np5EGkAlKujq4yge6R6fLoSxijmKIkEOMjU7n29q7rdo4vd4vsH9CluUT3kJkBAdwUMGDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T21:44:22.237663Z"},"content_sha256":"c530f4f97fd52706ffb0bd90f4891095939e3ef935989a00c84532f4479d1515","schema_version":"1.0","event_id":"sha256:c530f4f97fd52706ffb0bd90f4891095939e3ef935989a00c84532f4479d1515"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EQMLAMMOHY2VLU4E4DZE5QSGND","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Honesty to Subterfuge: In-Context Reinforcement Learning Can Make Honest Models Reward Hack","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Christoph Str\\\"ater, Joe Needham, Leo McKee-Reid, Maria Angelica Martinez, Mikita Balesni","submitted_at":"2024-10-09T02:34:27Z","abstract_excerpt":"Previous work has shown that training \"helpful-only\" LLMs with reinforcement learning on a curriculum of gameable environments can lead models to generalize to egregious specification gaming, such as editing their own reward function or modifying task checklists to appear more successful. We show that gpt-4o, gpt-4o-mini, o1-preview, and o1-mini - frontier models trained to be helpful, harmless, and honest - can engage in specification gaming without training on a curriculum of tasks, purely from in-context iterative reflection (which we call in-context reinforcement learning, \"ICRL\"). We also"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.06491","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.06491/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:18:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BMTfBt4PweXV0YlEfDOqdC8yodjMt7KNW1MeQ8dZ2xd76w79g/TvDR/g4KFCWVGvVx+uNipIa+Vf3trYt711Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T21:44:22.238099Z"},"content_sha256":"3990c3aea4f2bee9ebbcaa923fec7f2196b81cc248bfd5b988f3e667541fe5ca","schema_version":"1.0","event_id":"sha256:3990c3aea4f2bee9ebbcaa923fec7f2196b81cc248bfd5b988f3e667541fe5ca"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/bundle.json","state_url":"https://pith.science/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T21:44:22Z","links":{"resolver":"https://pith.science/pith/EQMLAMMOHY2VLU4E4DZE5QSGND","bundle":"https://pith.science/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/bundle.json","state":"https://pith.science/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EQMLAMMOHY2VLU4E4DZE5QSGND/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EQMLAMMOHY2VLU4E4DZE5QSGND","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"04914f7ccb547eefd23db4d14a0ce3ba614e6af6e11be670c31350bd6a9f2c65","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-09T02:34:27Z","title_canon_sha256":"07aa5f6d09d207054f8885edd72e3628de2efc493b2be1617433e4cbd71df53a"},"schema_version":"1.0","source":{"id":"2410.06491","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.06491","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"arxiv_version","alias_value":"2410.06491v1","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.06491","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_12","alias_value":"EQMLAMMOHY2V","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_16","alias_value":"EQMLAMMOHY2VLU4E","created_at":"2026-07-05T09:18:02Z"},{"alias_kind":"pith_short_8","alias_value":"EQMLAMMO","created_at":"2026-07-05T09:18:02Z"}],"graph_snapshots":[{"event_id":"sha256:3990c3aea4f2bee9ebbcaa923fec7f2196b81cc248bfd5b988f3e667541fe5ca","target":"graph","created_at":"2026-07-05T09:18:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.06491/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Previous work has shown that training \"helpful-only\" LLMs with reinforcement learning on a curriculum of gameable environments can lead models to generalize to egregious specification gaming, such as editing their own reward function or modifying task checklists to appear more successful. We show that gpt-4o, gpt-4o-mini, o1-preview, and o1-mini - frontier models trained to be helpful, harmless, and honest - can engage in specification gaming without training on a curriculum of tasks, purely from in-context iterative reflection (which we call in-context reinforcement learning, \"ICRL\"). We also","authors_text":"Christoph Str\\\"ater, Joe Needham, Leo McKee-Reid, Maria Angelica Martinez, Mikita Balesni","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-09T02:34:27Z","title":"Honesty to Subterfuge: In-Context Reinforcement Learning Can Make Honest Models Reward Hack"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.06491","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c530f4f97fd52706ffb0bd90f4891095939e3ef935989a00c84532f4479d1515","target":"record","created_at":"2026-07-05T09:18:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"04914f7ccb547eefd23db4d14a0ce3ba614e6af6e11be670c31350bd6a9f2c65","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-09T02:34:27Z","title_canon_sha256":"07aa5f6d09d207054f8885edd72e3628de2efc493b2be1617433e4cbd71df53a"},"schema_version":"1.0","source":{"id":"2410.06491","kind":"arxiv","version":1}},"canonical_sha256":"2418b0318e3e3555d384e0f24ec24668d0cdb344f2420a4682d4c524f6986da5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2418b0318e3e3555d384e0f24ec24668d0cdb344f2420a4682d4c524f6986da5","first_computed_at":"2026-07-05T09:18:02.529073Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:18:02.529073Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"c8iDpwFsxHgj51hpcnOv2POpPydOH/8+kvPBiQ4mcEUvLb4SUy4z1C1iY8tLkKBr+dvoZmYevieZTcMFJVq4Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:18:02.529425Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.06491","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c530f4f97fd52706ffb0bd90f4891095939e3ef935989a00c84532f4479d1515","sha256:3990c3aea4f2bee9ebbcaa923fec7f2196b81cc248bfd5b988f3e667541fe5ca"],"state_sha256":"fa116cac3aa594df0967f5f8e67edebbfe6d61c82656800375212cfde911cf2a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PDmWpjl0423PQ93vf46OTvg3mUMAI4zrFgt1CQGpZpjas6eR7lpWeykN+vcs4i02Km54Pcyc3CPKQs4PbMGzCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T21:44:22.240719Z","bundle_sha256":"50cbca43926f61a78bbc741bcff4e4cf2c822f9ccaa4d95c92ca9137bd054521"}}