{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:M4BT7W2BBI4DMDL2TPD4STC5OF","short_pith_number":"pith:M4BT7W2B","canonical_record":{"source":{"id":"2410.05782","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-08T08:04:09Z","cross_cats_sorted":[],"title_canon_sha256":"f15ec2349a47b1837012f6fabc61835cf73cfdc1adbb31e4113d90c68e82c665","abstract_canon_sha256":"0a7292ea1812e8c37934f628bdcd90108dc947fa8c7fd65c48c5809d63021a98"},"schema_version":"1.0"},"canonical_sha256":"67033fdb410a38360d7a9bc7c94c5d71539dafacbf5e9b5e2e266158947e49d1","source":{"kind":"arxiv","id":"2410.05782","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05782","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05782v1","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05782","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_12","alias_value":"M4BT7W2BBI4D","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_16","alias_value":"M4BT7W2BBI4DMDL2","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_8","alias_value":"M4BT7W2B","created_at":"2026-07-05T09:17:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:M4BT7W2BBI4DMDL2TPD4STC5OF","target":"record","payload":{"canonical_record":{"source":{"id":"2410.05782","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-08T08:04:09Z","cross_cats_sorted":[],"title_canon_sha256":"f15ec2349a47b1837012f6fabc61835cf73cfdc1adbb31e4113d90c68e82c665","abstract_canon_sha256":"0a7292ea1812e8c37934f628bdcd90108dc947fa8c7fd65c48c5809d63021a98"},"schema_version":"1.0"},"canonical_sha256":"67033fdb410a38360d7a9bc7c94c5d71539dafacbf5e9b5e2e266158947e49d1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:17:24.679215Z","signature_b64":"UtwDaKymOTqBKAJDK2L4/2/XqDPd1Bhj5/bhca0/KhcSljAUBp19+tGktzPzpgHIifPAoIp4ynRndrQjKAABAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"67033fdb410a38360d7a9bc7c94c5d71539dafacbf5e9b5e2e266158947e49d1","last_reissued_at":"2026-07-05T09:17:24.678735Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:17:24.678735Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.05782","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6pa29Xwa+Df4qy7N+6Lrf/AwEMKPJZ/+1OO90aB4tTvBjKEGjea+n8Gq/w/j//FU30Ec4aKYPjja0A8E4RhQAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:11:29.655681Z"},"content_sha256":"0a3fd8752dd24c908213d225314b30ac608f5cb0df940fd0055ca90a82913f85","schema_version":"1.0","event_id":"sha256:0a3fd8752dd24c908213d225314b30ac608f5cb0df940fd0055ca90a82913f85"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:M4BT7W2BBI4DMDL2TPD4STC5OF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reinforcement Learning From Imperfect Corrective Actions And Proxy Rewards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Changjie Fan, Paul Weng, Tangjie Lv, Tianze Zhou, Xuening Feng, Yan Song, Yifei Zhu, Yujing Hu, Zhaohui Jiang","submitted_at":"2024-10-08T08:04:09Z","abstract_excerpt":"In practice, reinforcement learning (RL) agents are often trained with a possibly imperfect proxy reward function, which may lead to a human-agent alignment issue (i.e., the learned policy either converges to non-optimal performance with low cumulative rewards, or achieves high cumulative rewards but in undesired manner). To tackle this issue, we consider a framework where a human labeler can provide additional feedback in the form of corrective actions, which expresses the labeler's action preferences although this feedback may possibly be imperfect as well. In this setting, to obtain a bette"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05782","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.05782/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hi+WeFmoscMwePmq8HMahDrmvtLjCgMlHZruflFttBfjvEZdt9SVhBk7gAWTzYNZ9jDmhAUM8rs3Vs+DSOCuDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:11:29.656127Z"},"content_sha256":"5eb33d6e92cb7b0b269265131b22a074faf9fd0ea672a541397a096b2fa01759","schema_version":"1.0","event_id":"sha256:5eb33d6e92cb7b0b269265131b22a074faf9fd0ea672a541397a096b2fa01759"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/bundle.json","state_url":"https://pith.science/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T03:11:29Z","links":{"resolver":"https://pith.science/pith/M4BT7W2BBI4DMDL2TPD4STC5OF","bundle":"https://pith.science/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/bundle.json","state":"https://pith.science/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/M4BT7W2BBI4DMDL2TPD4STC5OF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:M4BT7W2BBI4DMDL2TPD4STC5OF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0a7292ea1812e8c37934f628bdcd90108dc947fa8c7fd65c48c5809d63021a98","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-08T08:04:09Z","title_canon_sha256":"f15ec2349a47b1837012f6fabc61835cf73cfdc1adbb31e4113d90c68e82c665"},"schema_version":"1.0","source":{"id":"2410.05782","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05782","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05782v1","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05782","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_12","alias_value":"M4BT7W2BBI4D","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_16","alias_value":"M4BT7W2BBI4DMDL2","created_at":"2026-07-05T09:17:24Z"},{"alias_kind":"pith_short_8","alias_value":"M4BT7W2B","created_at":"2026-07-05T09:17:24Z"}],"graph_snapshots":[{"event_id":"sha256:5eb33d6e92cb7b0b269265131b22a074faf9fd0ea672a541397a096b2fa01759","target":"graph","created_at":"2026-07-05T09:17:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.05782/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In practice, reinforcement learning (RL) agents are often trained with a possibly imperfect proxy reward function, which may lead to a human-agent alignment issue (i.e., the learned policy either converges to non-optimal performance with low cumulative rewards, or achieves high cumulative rewards but in undesired manner). To tackle this issue, we consider a framework where a human labeler can provide additional feedback in the form of corrective actions, which expresses the labeler's action preferences although this feedback may possibly be imperfect as well. In this setting, to obtain a bette","authors_text":"Changjie Fan, Paul Weng, Tangjie Lv, Tianze Zhou, Xuening Feng, Yan Song, Yifei Zhu, Yujing Hu, Zhaohui Jiang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-08T08:04:09Z","title":"Reinforcement Learning From Imperfect Corrective Actions And Proxy Rewards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05782","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0a3fd8752dd24c908213d225314b30ac608f5cb0df940fd0055ca90a82913f85","target":"record","created_at":"2026-07-05T09:17:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0a7292ea1812e8c37934f628bdcd90108dc947fa8c7fd65c48c5809d63021a98","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-08T08:04:09Z","title_canon_sha256":"f15ec2349a47b1837012f6fabc61835cf73cfdc1adbb31e4113d90c68e82c665"},"schema_version":"1.0","source":{"id":"2410.05782","kind":"arxiv","version":1}},"canonical_sha256":"67033fdb410a38360d7a9bc7c94c5d71539dafacbf5e9b5e2e266158947e49d1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"67033fdb410a38360d7a9bc7c94c5d71539dafacbf5e9b5e2e266158947e49d1","first_computed_at":"2026-07-05T09:17:24.678735Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:17:24.678735Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UtwDaKymOTqBKAJDK2L4/2/XqDPd1Bhj5/bhca0/KhcSljAUBp19+tGktzPzpgHIifPAoIp4ynRndrQjKAABAA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:17:24.679215Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.05782","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0a3fd8752dd24c908213d225314b30ac608f5cb0df940fd0055ca90a82913f85","sha256:5eb33d6e92cb7b0b269265131b22a074faf9fd0ea672a541397a096b2fa01759"],"state_sha256":"e5e2a27b84d52076e6779b2a13b4435634c6556aa5b60d92d9b3d7638308b915"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4T88pgINWGOACTNkzhrVFcdFCAKzztsems+xlm2n9aUbtPL3y9vmw2KI54X+eAnIDHVEsvq4A1ylcKVoNfNYAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T03:11:29.658369Z","bundle_sha256":"f93d885a3cea924e0d168f25407905c0dd3b461e8c482f9e0ff13c285cea3652"}}