{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:HL2TDXVQBE5N7JY6JDXXIEV2IS","short_pith_number":"pith:HL2TDXVQ","canonical_record":{"source":{"id":"1808.04096","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-08-13T08:12:22Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"900ca7e36bb1b43d8212a9862cd487b6b264880e8ca9e3c11b8ed13b32898378","abstract_canon_sha256":"26a6985f9486a280595de57f82a8dced29ebe65ff0342e4dfaff74436f0ccccb"},"schema_version":"1.0"},"canonical_sha256":"3af531deb0093adfa71e48ef7412ba44baaa73ddc4dec0de67ac4c6a0ba79dcb","source":{"kind":"arxiv","id":"1808.04096","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1808.04096","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"arxiv_version","alias_value":"1808.04096v1","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1808.04096","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"pith_short_12","alias_value":"HL2TDXVQBE5N","created_at":"2026-05-18T12:32:28Z"},{"alias_kind":"pith_short_16","alias_value":"HL2TDXVQBE5N7JY6","created_at":"2026-05-18T12:32:28Z"},{"alias_kind":"pith_short_8","alias_value":"HL2TDXVQ","created_at":"2026-05-18T12:32:28Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:HL2TDXVQBE5N7JY6JDXXIEV2IS","target":"record","payload":{"canonical_record":{"source":{"id":"1808.04096","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-08-13T08:12:22Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"900ca7e36bb1b43d8212a9862cd487b6b264880e8ca9e3c11b8ed13b32898378","abstract_canon_sha256":"26a6985f9486a280595de57f82a8dced29ebe65ff0342e4dfaff74436f0ccccb"},"schema_version":"1.0"},"canonical_sha256":"3af531deb0093adfa71e48ef7412ba44baaa73ddc4dec0de67ac4c6a0ba79dcb","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:08:18.892321Z","signature_b64":"hTLuObUDPigaxeWEWOngAb9HgytwTVu3PDL/ZbSC4I5E81YFFDITF2gdbYCcoqNBCzRLOgZavQ6jt15c/2lXBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3af531deb0093adfa71e48ef7412ba44baaa73ddc4dec0de67ac4c6a0ba79dcb","last_reissued_at":"2026-05-18T00:08:18.891903Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:08:18.891903Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1808.04096","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:08:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oVP6C4vVJr1eiLUMBqdU6xc+hagFCKeClg3wGVwW4cV3vxFokJRaIDLQ+VmUy4SSJteHS9MQOpRriRHTUr8VCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-06T08:56:55.628548Z"},"content_sha256":"df70e0cd9d7e015ac2c007bf4453161104df17858de9cf286e4223cb7e835acd","schema_version":"1.0","event_id":"sha256:df70e0cd9d7e015ac2c007bf4453161104df17858de9cf286e4223cb7e835acd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:HL2TDXVQBE5N7JY6JDXXIEV2IS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Directed Policy Gradient for Safe Reinforcement Learning with Human Advice","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Ann Now\\'e, Denis Steckelmacher, Diederik M. Roijers, H\\'el\\`ene Plisnier, Tim Brys","submitted_at":"2018-08-13T08:12:22Z","abstract_excerpt":"Many currently deployed Reinforcement Learning agents work in an environment shared with humans, be them co-workers, users or clients. It is desirable that these agents adjust to people's preferences, learn faster thanks to their help, and act safely around them. We argue that most current approaches that learn from human feedback are unsafe: rewarding or punishing the agent a-posteriori cannot immediately prevent it from wrong-doing. In this paper, we extend Policy Gradient to make it robust to external directives, that would otherwise break the fundamentally on-policy nature of Policy Gradie"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1808.04096","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:08:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"unVIAl/PcXLnUhlxbyukJMLUSEHuVx2T20rtHUVaI3cjPew7TtHy4suJ3cT3ZLZO3K1/+g6Op04BZaDQOW1BDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-06T08:56:55.628891Z"},"content_sha256":"37c19dc05ac475ab6dbc2e51cadf88ddff5bfafad2cc0c23b46fbb39160a35d6","schema_version":"1.0","event_id":"sha256:37c19dc05ac475ab6dbc2e51cadf88ddff5bfafad2cc0c23b46fbb39160a35d6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/bundle.json","state_url":"https://pith.science/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-06T08:56:55Z","links":{"resolver":"https://pith.science/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS","bundle":"https://pith.science/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/bundle.json","state":"https://pith.science/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HL2TDXVQBE5N7JY6JDXXIEV2IS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:HL2TDXVQBE5N7JY6JDXXIEV2IS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"26a6985f9486a280595de57f82a8dced29ebe65ff0342e4dfaff74436f0ccccb","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-08-13T08:12:22Z","title_canon_sha256":"900ca7e36bb1b43d8212a9862cd487b6b264880e8ca9e3c11b8ed13b32898378"},"schema_version":"1.0","source":{"id":"1808.04096","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1808.04096","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"arxiv_version","alias_value":"1808.04096v1","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1808.04096","created_at":"2026-05-18T00:08:18Z"},{"alias_kind":"pith_short_12","alias_value":"HL2TDXVQBE5N","created_at":"2026-05-18T12:32:28Z"},{"alias_kind":"pith_short_16","alias_value":"HL2TDXVQBE5N7JY6","created_at":"2026-05-18T12:32:28Z"},{"alias_kind":"pith_short_8","alias_value":"HL2TDXVQ","created_at":"2026-05-18T12:32:28Z"}],"graph_snapshots":[{"event_id":"sha256:37c19dc05ac475ab6dbc2e51cadf88ddff5bfafad2cc0c23b46fbb39160a35d6","target":"graph","created_at":"2026-05-18T00:08:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Many currently deployed Reinforcement Learning agents work in an environment shared with humans, be them co-workers, users or clients. It is desirable that these agents adjust to people's preferences, learn faster thanks to their help, and act safely around them. We argue that most current approaches that learn from human feedback are unsafe: rewarding or punishing the agent a-posteriori cannot immediately prevent it from wrong-doing. In this paper, we extend Policy Gradient to make it robust to external directives, that would otherwise break the fundamentally on-policy nature of Policy Gradie","authors_text":"Ann Now\\'e, Denis Steckelmacher, Diederik M. Roijers, H\\'el\\`ene Plisnier, Tim Brys","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-08-13T08:12:22Z","title":"Directed Policy Gradient for Safe Reinforcement Learning with Human Advice"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1808.04096","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:df70e0cd9d7e015ac2c007bf4453161104df17858de9cf286e4223cb7e835acd","target":"record","created_at":"2026-05-18T00:08:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"26a6985f9486a280595de57f82a8dced29ebe65ff0342e4dfaff74436f0ccccb","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-08-13T08:12:22Z","title_canon_sha256":"900ca7e36bb1b43d8212a9862cd487b6b264880e8ca9e3c11b8ed13b32898378"},"schema_version":"1.0","source":{"id":"1808.04096","kind":"arxiv","version":1}},"canonical_sha256":"3af531deb0093adfa71e48ef7412ba44baaa73ddc4dec0de67ac4c6a0ba79dcb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3af531deb0093adfa71e48ef7412ba44baaa73ddc4dec0de67ac4c6a0ba79dcb","first_computed_at":"2026-05-18T00:08:18.891903Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:08:18.891903Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"hTLuObUDPigaxeWEWOngAb9HgytwTVu3PDL/ZbSC4I5E81YFFDITF2gdbYCcoqNBCzRLOgZavQ6jt15c/2lXBg==","signature_status":"signed_v1","signed_at":"2026-05-18T00:08:18.892321Z","signed_message":"canonical_sha256_bytes"},"source_id":"1808.04096","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:df70e0cd9d7e015ac2c007bf4453161104df17858de9cf286e4223cb7e835acd","sha256:37c19dc05ac475ab6dbc2e51cadf88ddff5bfafad2cc0c23b46fbb39160a35d6"],"state_sha256":"b928e17ac6c1b032b4868cbae5ce7ff3a6969cfeb5386785494b73c18b348a8a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bPb94gv9ZEudZZeJoMV2qTaATjVnMQ06Huqo79rCimjeN3Duj52MLYYJ02PC/KIhQDTQkvbNclYVEOkFNZg4AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-06T08:56:55.770385Z","bundle_sha256":"fb53b44d856e74af3a15b969d712bcc7d893e7bdd863d1deab28a03af55f00c2"}}