{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:NOSWDOLDJLFJIFHSPTI4WK76WZ","short_pith_number":"pith:NOSWDOLD","canonical_record":{"source":{"id":"2112.12228","kind":"arxiv","version":6},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-12-22T21:12:28Z","cross_cats_sorted":[],"title_canon_sha256":"988b76e03438660b8484b218e092af2824023a7e646e62b2f9195be60c7fd9d0","abstract_canon_sha256":"7d81469cbd868677f159f6c0d9422978f1988abcd53e8a2b985026f4586dd859"},"schema_version":"1.0"},"canonical_sha256":"6ba561b9634aca9414f27cd1cb2bfeb6781461f4dc27cfaed45fe74b413ba486","source":{"kind":"arxiv","id":"2112.12228","version":6},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.12228","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"arxiv_version","alias_value":"2112.12228v6","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.12228","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_12","alias_value":"NOSWDOLDJLFJ","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_16","alias_value":"NOSWDOLDJLFJIFHS","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_8","alias_value":"NOSWDOLD","created_at":"2026-07-05T04:32:54Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:NOSWDOLDJLFJIFHSPTI4WK76WZ","target":"record","payload":{"canonical_record":{"source":{"id":"2112.12228","kind":"arxiv","version":6},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-12-22T21:12:28Z","cross_cats_sorted":[],"title_canon_sha256":"988b76e03438660b8484b218e092af2824023a7e646e62b2f9195be60c7fd9d0","abstract_canon_sha256":"7d81469cbd868677f159f6c0d9422978f1988abcd53e8a2b985026f4586dd859"},"schema_version":"1.0"},"canonical_sha256":"6ba561b9634aca9414f27cd1cb2bfeb6781461f4dc27cfaed45fe74b413ba486","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:32:54.216345Z","signature_b64":"1zdzj5AqpsAVputUjBi1o3vnjRqchBYeo6K7ARu/GuBw9H3AthXofMvl9XeX//fxxrLY5PHDio+TM/sOJmNqDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6ba561b9634aca9414f27cd1cb2bfeb6781461f4dc27cfaed45fe74b413ba486","last_reissued_at":"2026-07-05T04:32:54.215925Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:32:54.215925Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2112.12228","source_version":6,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:32:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/Q7enmrQjKokJJWRqKFuDvFN6F1I4U2BL7UeXqD8KZPYqqou5C6zb4Qy2l1jmieb1W6Syb7LFoOZt1FpxzoJBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T09:07:11.021504Z"},"content_sha256":"11be74b1b7b6c773ac9e99bcc8e19af27811c0a0596b4f35b500d91a00b6038b","schema_version":"1.0","event_id":"sha256:11be74b1b7b6c773ac9e99bcc8e19af27811c0a0596b4f35b500d91a00b6038b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:NOSWDOLDJLFJIFHSPTI4WK76WZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Direct Behavior Specification via Constrained Reinforcement Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Christopher Pal, Joshua Romoff, Julien Roy, Pierre-Luc Bacon, Roger Girgis","submitted_at":"2021-12-22T21:12:28Z","abstract_excerpt":"The standard formulation of Reinforcement Learning lacks a practical way of specifying what are admissible and forbidden behaviors. Most often, practitioners go about the task of behavior specification by manually engineering the reward function, a counter-intuitive process that requires several iterations and is prone to reward hacking by the agent. In this work, we argue that constrained RL, which has almost exclusively been used for safe RL, also has the potential to significantly reduce the amount of work spent for reward specification in applied RL projects. To this end, we propose to spe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.12228","kind":"arxiv","version":6},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2112.12228/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:32:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0+OVF76y7vd6WAkkTJEmIFW83PVDtbUFdaH7jpOTiicOOa9eq+vnjOafHsEzdlW+TmYvX3Ya5Y01S5wcHZEDDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T09:07:11.022121Z"},"content_sha256":"1d36423753011d2f5af0dc11460d9e5d305169a03841bf1b443bf8135397d3c7","schema_version":"1.0","event_id":"sha256:1d36423753011d2f5af0dc11460d9e5d305169a03841bf1b443bf8135397d3c7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/bundle.json","state_url":"https://pith.science/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T09:07:11Z","links":{"resolver":"https://pith.science/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ","bundle":"https://pith.science/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/bundle.json","state":"https://pith.science/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NOSWDOLDJLFJIFHSPTI4WK76WZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:NOSWDOLDJLFJIFHSPTI4WK76WZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7d81469cbd868677f159f6c0d9422978f1988abcd53e8a2b985026f4586dd859","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-12-22T21:12:28Z","title_canon_sha256":"988b76e03438660b8484b218e092af2824023a7e646e62b2f9195be60c7fd9d0"},"schema_version":"1.0","source":{"id":"2112.12228","kind":"arxiv","version":6}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.12228","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"arxiv_version","alias_value":"2112.12228v6","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.12228","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_12","alias_value":"NOSWDOLDJLFJ","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_16","alias_value":"NOSWDOLDJLFJIFHS","created_at":"2026-07-05T04:32:54Z"},{"alias_kind":"pith_short_8","alias_value":"NOSWDOLD","created_at":"2026-07-05T04:32:54Z"}],"graph_snapshots":[{"event_id":"sha256:1d36423753011d2f5af0dc11460d9e5d305169a03841bf1b443bf8135397d3c7","target":"graph","created_at":"2026-07-05T04:32:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2112.12228/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The standard formulation of Reinforcement Learning lacks a practical way of specifying what are admissible and forbidden behaviors. Most often, practitioners go about the task of behavior specification by manually engineering the reward function, a counter-intuitive process that requires several iterations and is prone to reward hacking by the agent. In this work, we argue that constrained RL, which has almost exclusively been used for safe RL, also has the potential to significantly reduce the amount of work spent for reward specification in applied RL projects. To this end, we propose to spe","authors_text":"Christopher Pal, Joshua Romoff, Julien Roy, Pierre-Luc Bacon, Roger Girgis","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-12-22T21:12:28Z","title":"Direct Behavior Specification via Constrained Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.12228","kind":"arxiv","version":6},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:11be74b1b7b6c773ac9e99bcc8e19af27811c0a0596b4f35b500d91a00b6038b","target":"record","created_at":"2026-07-05T04:32:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7d81469cbd868677f159f6c0d9422978f1988abcd53e8a2b985026f4586dd859","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-12-22T21:12:28Z","title_canon_sha256":"988b76e03438660b8484b218e092af2824023a7e646e62b2f9195be60c7fd9d0"},"schema_version":"1.0","source":{"id":"2112.12228","kind":"arxiv","version":6}},"canonical_sha256":"6ba561b9634aca9414f27cd1cb2bfeb6781461f4dc27cfaed45fe74b413ba486","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6ba561b9634aca9414f27cd1cb2bfeb6781461f4dc27cfaed45fe74b413ba486","first_computed_at":"2026-07-05T04:32:54.215925Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:32:54.215925Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1zdzj5AqpsAVputUjBi1o3vnjRqchBYeo6K7ARu/GuBw9H3AthXofMvl9XeX//fxxrLY5PHDio+TM/sOJmNqDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:32:54.216345Z","signed_message":"canonical_sha256_bytes"},"source_id":"2112.12228","source_kind":"arxiv","source_version":6}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:11be74b1b7b6c773ac9e99bcc8e19af27811c0a0596b4f35b500d91a00b6038b","sha256:1d36423753011d2f5af0dc11460d9e5d305169a03841bf1b443bf8135397d3c7"],"state_sha256":"047f873776e4434a0922e9760eaab6ccc799dc785149d45ba56a77180c539f76"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"n9xBNQJszzGG5C1+aiyjtxS4aF+nyWP8NVYDgeXUBIzGt5SK7tZ/VWtP2NVMzXO/v1Z3IjGLsdy+zN4xvP0fCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T09:07:11.025273Z","bundle_sha256":"0c1030a1202353fceb7d282763427d416311f79f3bad93ea361bcb3ee6ce3151"}}