{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:3DGP6SJKRFVF2WSSD3JT5FXT2A","short_pith_number":"pith:3DGP6SJK","canonical_record":{"source":{"id":"2301.03652","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-01-09T19:45:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"8896e9bf0da86ff90b4cfada5f5f8fec21dd44620976360f87c6cad24a82b79c","abstract_canon_sha256":"dde3a4abc1233a75b0eb5e9a7999f8b0f97a20ad13e24ba3f501e96b8d5639e8"},"schema_version":"1.0"},"canonical_sha256":"d8ccff492a896a5d5a521ed33e96f3d03086cf9355200ad395c4d4196be4085d","source":{"kind":"arxiv","id":"2301.03652","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2301.03652","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"arxiv_version","alias_value":"2301.03652v1","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2301.03652","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_12","alias_value":"3DGP6SJKRFVF","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_16","alias_value":"3DGP6SJKRFVF2WSS","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_8","alias_value":"3DGP6SJK","created_at":"2026-07-05T05:32:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:3DGP6SJKRFVF2WSSD3JT5FXT2A","target":"record","payload":{"canonical_record":{"source":{"id":"2301.03652","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-01-09T19:45:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"8896e9bf0da86ff90b4cfada5f5f8fec21dd44620976360f87c6cad24a82b79c","abstract_canon_sha256":"dde3a4abc1233a75b0eb5e9a7999f8b0f97a20ad13e24ba3f501e96b8d5639e8"},"schema_version":"1.0"},"canonical_sha256":"d8ccff492a896a5d5a521ed33e96f3d03086cf9355200ad395c4d4196be4085d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:32:04.189071Z","signature_b64":"oWi3YdAqpYiMKLIyIbSm4MabB3SsDGGTKpa+/m6J2itZbAunvHtTwQrQ4EOaak6FdkTbnY4JlR0kZjuVYoBDDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d8ccff492a896a5d5a521ed33e96f3d03086cf9355200ad395c4d4196be4085d","last_reissued_at":"2026-07-05T05:32:04.188646Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:32:04.188646Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2301.03652","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:32:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QN3pBWbUKBdEEALYNThbTHzf56iPWdOeIx7qxNub6I3xlUEGDe2mBWEcdPyuJ7z6uahY0x8ZiVx0ET9lYlxDDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T21:03:59.799311Z"},"content_sha256":"559cd5d5e9dd6a340bd5b44cd7e77d0678f320856958625cc9c9ee2fd69185fb","schema_version":"1.0","event_id":"sha256:559cd5d5e9dd6a340bd5b44cd7e77d0678f320856958625cc9c9ee2fd69185fb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:3DGP6SJKRFVF2WSSD3JT5FXT2A","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On The Fragility of Learned Reward Functions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Adam Gleave, David Krueger, Lev McKinney, Yawen Duan","submitted_at":"2023-01-09T19:45:38Z","abstract_excerpt":"Reward functions are notoriously difficult to specify, especially for tasks with complex goals. Reward learning approaches attempt to infer reward functions from human feedback and preferences. Prior works on reward learning have mainly focused on the performance of policies trained alongside the reward function. This practice, however, may fail to detect learned rewards that are not capable of training new policies from scratch and thus do not capture the intended behavior. Our work focuses on demonstrating and studying the causes of these relearning failures in the domain of preference-based"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2301.03652","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2301.03652/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:32:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HXK9xetKIkec5H6hv9MYF67FkSaRT6WVM6S3dcdamyMDrhg/gaqv1Rmg744l/Z8CrhI7HgtvF+SydprVPzi/Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T21:03:59.799934Z"},"content_sha256":"a109b16b565d910b295cae5beedfadce29bb3e518a77c40bcf639a0210729cf8","schema_version":"1.0","event_id":"sha256:a109b16b565d910b295cae5beedfadce29bb3e518a77c40bcf639a0210729cf8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/bundle.json","state_url":"https://pith.science/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T21:03:59Z","links":{"resolver":"https://pith.science/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A","bundle":"https://pith.science/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/bundle.json","state":"https://pith.science/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3DGP6SJKRFVF2WSSD3JT5FXT2A/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:3DGP6SJKRFVF2WSSD3JT5FXT2A","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dde3a4abc1233a75b0eb5e9a7999f8b0f97a20ad13e24ba3f501e96b8d5639e8","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-01-09T19:45:38Z","title_canon_sha256":"8896e9bf0da86ff90b4cfada5f5f8fec21dd44620976360f87c6cad24a82b79c"},"schema_version":"1.0","source":{"id":"2301.03652","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2301.03652","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"arxiv_version","alias_value":"2301.03652v1","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2301.03652","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_12","alias_value":"3DGP6SJKRFVF","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_16","alias_value":"3DGP6SJKRFVF2WSS","created_at":"2026-07-05T05:32:04Z"},{"alias_kind":"pith_short_8","alias_value":"3DGP6SJK","created_at":"2026-07-05T05:32:04Z"}],"graph_snapshots":[{"event_id":"sha256:a109b16b565d910b295cae5beedfadce29bb3e518a77c40bcf639a0210729cf8","target":"graph","created_at":"2026-07-05T05:32:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2301.03652/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reward functions are notoriously difficult to specify, especially for tasks with complex goals. Reward learning approaches attempt to infer reward functions from human feedback and preferences. Prior works on reward learning have mainly focused on the performance of policies trained alongside the reward function. This practice, however, may fail to detect learned rewards that are not capable of training new policies from scratch and thus do not capture the intended behavior. Our work focuses on demonstrating and studying the causes of these relearning failures in the domain of preference-based","authors_text":"Adam Gleave, David Krueger, Lev McKinney, Yawen Duan","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-01-09T19:45:38Z","title":"On The Fragility of Learned Reward Functions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2301.03652","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:559cd5d5e9dd6a340bd5b44cd7e77d0678f320856958625cc9c9ee2fd69185fb","target":"record","created_at":"2026-07-05T05:32:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dde3a4abc1233a75b0eb5e9a7999f8b0f97a20ad13e24ba3f501e96b8d5639e8","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-01-09T19:45:38Z","title_canon_sha256":"8896e9bf0da86ff90b4cfada5f5f8fec21dd44620976360f87c6cad24a82b79c"},"schema_version":"1.0","source":{"id":"2301.03652","kind":"arxiv","version":1}},"canonical_sha256":"d8ccff492a896a5d5a521ed33e96f3d03086cf9355200ad395c4d4196be4085d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d8ccff492a896a5d5a521ed33e96f3d03086cf9355200ad395c4d4196be4085d","first_computed_at":"2026-07-05T05:32:04.188646Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:32:04.188646Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"oWi3YdAqpYiMKLIyIbSm4MabB3SsDGGTKpa+/m6J2itZbAunvHtTwQrQ4EOaak6FdkTbnY4JlR0kZjuVYoBDDg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:32:04.189071Z","signed_message":"canonical_sha256_bytes"},"source_id":"2301.03652","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:559cd5d5e9dd6a340bd5b44cd7e77d0678f320856958625cc9c9ee2fd69185fb","sha256:a109b16b565d910b295cae5beedfadce29bb3e518a77c40bcf639a0210729cf8"],"state_sha256":"871eaaa4e0676ada4d73e204141d639a7935d0953a8c532f5fb2de73e41be3e9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H/1WXy3j/a1u3jGzibypGPzvNbOBxPJfd3E2/gFa3L/Cm5h9/O1pL9Er46w14QwKNMRZWJ3jh/Wc1bPzjPfJDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T21:03:59.804973Z","bundle_sha256":"0b2547e2a56a77a5b4fa1e62df7705c74be0fabd4efc41e5e2ee123db002d076"}}