{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:4HGJBNOYUMAYMPV2HHZC5RXRGO","short_pith_number":"pith:4HGJBNOY","canonical_record":{"source":{"id":"2105.12545","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2021-05-26T13:52:39Z","cross_cats_sorted":[],"title_canon_sha256":"d2f6334a3654b1ce829e917bcc32c92d13832d96d129120c378856723abaa4ec","abstract_canon_sha256":"1504db40c60f307f9d237d1d4113cd59b8d8f738164bf92f318c799d9c375ba8"},"schema_version":"1.0"},"canonical_sha256":"e1cc90b5d8a301863eba39f22ec6f133a320d68093699418fbc1b70243962819","source":{"kind":"arxiv","id":"2105.12545","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.12545","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"arxiv_version","alias_value":"2105.12545v1","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.12545","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_12","alias_value":"4HGJBNOYUMAY","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_16","alias_value":"4HGJBNOYUMAYMPV2","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_8","alias_value":"4HGJBNOY","created_at":"2026-07-05T04:15:42Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:4HGJBNOYUMAYMPV2HHZC5RXRGO","target":"record","payload":{"canonical_record":{"source":{"id":"2105.12545","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2021-05-26T13:52:39Z","cross_cats_sorted":[],"title_canon_sha256":"d2f6334a3654b1ce829e917bcc32c92d13832d96d129120c378856723abaa4ec","abstract_canon_sha256":"1504db40c60f307f9d237d1d4113cd59b8d8f738164bf92f318c799d9c375ba8"},"schema_version":"1.0"},"canonical_sha256":"e1cc90b5d8a301863eba39f22ec6f133a320d68093699418fbc1b70243962819","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:15:42.751824Z","signature_b64":"2dLM/Afi1u2NUBxX5HBTT0eCphs/LZuCn7xRzD0rI8jNkkoCAPypPZqgfV+FrfRqt2vI1CIA083zSH1NClhKDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e1cc90b5d8a301863eba39f22ec6f133a320d68093699418fbc1b70243962819","last_reissued_at":"2026-07-05T04:15:42.751337Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:15:42.751337Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2105.12545","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:15:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+hl/mHV6tZFo73/fR/KBuGunzoE5AVvQOfZ25NzqVOYIEtGjqlzFTufjGHCm3vGbzSHUbCVNB7KUMQsui6BlCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-20T00:00:15.468967Z"},"content_sha256":"3a437f40b989b9e65d39e366d75553c2c1bc6155ac4ee4a2450191de6557db5d","schema_version":"1.0","event_id":"sha256:3a437f40b989b9e65d39e366d75553c2c1bc6155ac4ee4a2450191de6557db5d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:4HGJBNOYUMAYMPV2HHZC5RXRGO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Successive Convex Approximation Based Off-Policy Optimization for Constrained Reinforcement Learning","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"An Liu, Chang Tian, Guang Huang, Wu Luo","submitted_at":"2021-05-26T13:52:39Z","abstract_excerpt":"We propose a successive convex approximation based off-policy optimization (SCAOPO) algorithm to solve the general constrained reinforcement learning problem, which is formulated as a constrained Markov decision process (CMDP) in the context of average cost. The SCAOPO is based on solving a sequence of convex objective/feasibility optimization problems obtained by replacing the objective and constraint functions in the original problems with convex surrogate functions. At each iteration, the convex surrogate problem can be efficiently solved by Lagrange dual method even the policy is parameter"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.12545","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.12545/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:15:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+TJD8WzqPt2u1wRMeBzzlMPYDalItgPJlX6Rig9XpTFUPPUi5UUgFlo5iscJKlNFWxZIgRelkh4JkRxh4RMTCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-20T00:00:15.469354Z"},"content_sha256":"dc2a5bb52e4c409ba0bdf4ffb2184d4991db0044ebafa88a02625812df6046b8","schema_version":"1.0","event_id":"sha256:dc2a5bb52e4c409ba0bdf4ffb2184d4991db0044ebafa88a02625812df6046b8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/bundle.json","state_url":"https://pith.science/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-20T00:00:15Z","links":{"resolver":"https://pith.science/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO","bundle":"https://pith.science/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/bundle.json","state":"https://pith.science/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4HGJBNOYUMAYMPV2HHZC5RXRGO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:4HGJBNOYUMAYMPV2HHZC5RXRGO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1504db40c60f307f9d237d1d4113cd59b8d8f738164bf92f318c799d9c375ba8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2021-05-26T13:52:39Z","title_canon_sha256":"d2f6334a3654b1ce829e917bcc32c92d13832d96d129120c378856723abaa4ec"},"schema_version":"1.0","source":{"id":"2105.12545","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.12545","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"arxiv_version","alias_value":"2105.12545v1","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.12545","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_12","alias_value":"4HGJBNOYUMAY","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_16","alias_value":"4HGJBNOYUMAYMPV2","created_at":"2026-07-05T04:15:42Z"},{"alias_kind":"pith_short_8","alias_value":"4HGJBNOY","created_at":"2026-07-05T04:15:42Z"}],"graph_snapshots":[{"event_id":"sha256:dc2a5bb52e4c409ba0bdf4ffb2184d4991db0044ebafa88a02625812df6046b8","target":"graph","created_at":"2026-07-05T04:15:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2105.12545/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose a successive convex approximation based off-policy optimization (SCAOPO) algorithm to solve the general constrained reinforcement learning problem, which is formulated as a constrained Markov decision process (CMDP) in the context of average cost. The SCAOPO is based on solving a sequence of convex objective/feasibility optimization problems obtained by replacing the objective and constraint functions in the original problems with convex surrogate functions. At each iteration, the convex surrogate problem can be efficiently solved by Lagrange dual method even the policy is parameter","authors_text":"An Liu, Chang Tian, Guang Huang, Wu Luo","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2021-05-26T13:52:39Z","title":"Successive Convex Approximation Based Off-Policy Optimization for Constrained Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.12545","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3a437f40b989b9e65d39e366d75553c2c1bc6155ac4ee4a2450191de6557db5d","target":"record","created_at":"2026-07-05T04:15:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1504db40c60f307f9d237d1d4113cd59b8d8f738164bf92f318c799d9c375ba8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2021-05-26T13:52:39Z","title_canon_sha256":"d2f6334a3654b1ce829e917bcc32c92d13832d96d129120c378856723abaa4ec"},"schema_version":"1.0","source":{"id":"2105.12545","kind":"arxiv","version":1}},"canonical_sha256":"e1cc90b5d8a301863eba39f22ec6f133a320d68093699418fbc1b70243962819","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e1cc90b5d8a301863eba39f22ec6f133a320d68093699418fbc1b70243962819","first_computed_at":"2026-07-05T04:15:42.751337Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:15:42.751337Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2dLM/Afi1u2NUBxX5HBTT0eCphs/LZuCn7xRzD0rI8jNkkoCAPypPZqgfV+FrfRqt2vI1CIA083zSH1NClhKDg==","signature_status":"signed_v1","signed_at":"2026-07-05T04:15:42.751824Z","signed_message":"canonical_sha256_bytes"},"source_id":"2105.12545","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3a437f40b989b9e65d39e366d75553c2c1bc6155ac4ee4a2450191de6557db5d","sha256:dc2a5bb52e4c409ba0bdf4ffb2184d4991db0044ebafa88a02625812df6046b8"],"state_sha256":"bdc58a7ad9b802f0f3afa46ac357d93d2dc8d392d58f7f6d08f81cde5fb05a32"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+i+sWuvwc0Ynz8ouZFkQX9stXXQfJ6p2/WtRWAwqF6tmTOZ12yaaJ/goj9sY/bhqOnrA+W2DMrVdsQfjdZJ2Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-20T00:00:15.471500Z","bundle_sha256":"363e6f6982785423fc6fdfd7b11fcc2126040d6f901ecba9e84164cea8310ddb"}}