{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2011:KBMVHNZIIFMQYREV4CHKE7ZNU4","short_pith_number":"pith:KBMVHNZI","canonical_record":{"source":{"id":"1110.4946","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"math.OC","submitted_at":"2011-10-22T04:54:34Z","cross_cats_sorted":[],"title_canon_sha256":"826ec5374fa49c873b4093118585e38c029a1bf145458c4b84864996d16696a2","abstract_canon_sha256":"0e2864338d181dd127242090c8c408f65800a8115c8217e8de854c7330abf60d"},"schema_version":"1.0"},"canonical_sha256":"505953b72841590c4495e08ea27f2da73663e550fcd8fb8dcf50c87ad639af8b","source":{"kind":"arxiv","id":"1110.4946","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1110.4946","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"arxiv_version","alias_value":"1110.4946v3","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1110.4946","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"pith_short_12","alias_value":"KBMVHNZIIFMQ","created_at":"2026-05-18T12:26:32Z"},{"alias_kind":"pith_short_16","alias_value":"KBMVHNZIIFMQYREV","created_at":"2026-05-18T12:26:32Z"},{"alias_kind":"pith_short_8","alias_value":"KBMVHNZI","created_at":"2026-05-18T12:26:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2011:KBMVHNZIIFMQYREV4CHKE7ZNU4","target":"record","payload":{"canonical_record":{"source":{"id":"1110.4946","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"math.OC","submitted_at":"2011-10-22T04:54:34Z","cross_cats_sorted":[],"title_canon_sha256":"826ec5374fa49c873b4093118585e38c029a1bf145458c4b84864996d16696a2","abstract_canon_sha256":"0e2864338d181dd127242090c8c408f65800a8115c8217e8de854c7330abf60d"},"schema_version":"1.0"},"canonical_sha256":"505953b72841590c4495e08ea27f2da73663e550fcd8fb8dcf50c87ad639af8b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:58:15.523194Z","signature_b64":"n0haXmNG5wE/AZjONFiqmbviGjJx7mP8XOc2Uha2xyiJIQiuGJs4YxVpEAi3cwEupZb4IbeyAotZOFSS+Uu4BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"505953b72841590c4495e08ea27f2da73663e550fcd8fb8dcf50c87ad639af8b","last_reissued_at":"2026-05-17T23:58:15.522664Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:58:15.522664Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1110.4946","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:58:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Rj6mdIgsvVthLYmDf08C2ktMHIla7FmKTTpaMFP94mtIHhzrmtOcWUjB67mDsxNWbX8zTWw2YujDVEyvVxZiCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T05:08:52.313633Z"},"content_sha256":"8761e333194337f29b3678c7cea753d566fe1a23c839a14b17a09366119f2825","schema_version":"1.0","event_id":"sha256:8761e333194337f29b3678c7cea753d566fe1a23c839a14b17a09366119f2825"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2011:KBMVHNZIIFMQYREV4CHKE7ZNU4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Real-Time Reinforcement Learning of Constrained Markov Decision Processes with Weak Derivatives","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"math.OC","authors_text":"Felisa Vazquez Abad, Vikram Krishnamurthy","submitted_at":"2011-10-22T04:54:34Z","abstract_excerpt":"We present on-line policy gradient algorithms for computing the locally optimal policy of a constrained, average cost, finite state Markov Decision Process. The stochastic approximation algorithms require estimation of the gradient of the cost function with respect to the parameter that characterizes the randomized policy. We propose a spherical coordinate parametrization and present a novel simulation based gradient estimation scheme involving weak derivatives (measure-valued differentiation). Such methods have substantially reduced variance compared to the widely used score function method. "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1110.4946","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:58:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"scArg8hcDE26oBmxNSjT849s04oboTk/ilDzdY0OvjFGtaoXmP1CU5BF9SC1BPahQ57XSPmzHtj5rfLceAkUBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T05:08:52.314091Z"},"content_sha256":"a78971179406896160b42308c35155ea18a4dc97d252746787763bd34f72e5a5","schema_version":"1.0","event_id":"sha256:a78971179406896160b42308c35155ea18a4dc97d252746787763bd34f72e5a5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/bundle.json","state_url":"https://pith.science/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-23T05:08:52Z","links":{"resolver":"https://pith.science/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4","bundle":"https://pith.science/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/bundle.json","state":"https://pith.science/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KBMVHNZIIFMQYREV4CHKE7ZNU4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2011:KBMVHNZIIFMQYREV4CHKE7ZNU4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0e2864338d181dd127242090c8c408f65800a8115c8217e8de854c7330abf60d","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"math.OC","submitted_at":"2011-10-22T04:54:34Z","title_canon_sha256":"826ec5374fa49c873b4093118585e38c029a1bf145458c4b84864996d16696a2"},"schema_version":"1.0","source":{"id":"1110.4946","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1110.4946","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"arxiv_version","alias_value":"1110.4946v3","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1110.4946","created_at":"2026-05-17T23:58:15Z"},{"alias_kind":"pith_short_12","alias_value":"KBMVHNZIIFMQ","created_at":"2026-05-18T12:26:32Z"},{"alias_kind":"pith_short_16","alias_value":"KBMVHNZIIFMQYREV","created_at":"2026-05-18T12:26:32Z"},{"alias_kind":"pith_short_8","alias_value":"KBMVHNZI","created_at":"2026-05-18T12:26:32Z"}],"graph_snapshots":[{"event_id":"sha256:a78971179406896160b42308c35155ea18a4dc97d252746787763bd34f72e5a5","target":"graph","created_at":"2026-05-17T23:58:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"We present on-line policy gradient algorithms for computing the locally optimal policy of a constrained, average cost, finite state Markov Decision Process. The stochastic approximation algorithms require estimation of the gradient of the cost function with respect to the parameter that characterizes the randomized policy. We propose a spherical coordinate parametrization and present a novel simulation based gradient estimation scheme involving weak derivatives (measure-valued differentiation). Such methods have substantially reduced variance compared to the widely used score function method. ","authors_text":"Felisa Vazquez Abad, Vikram Krishnamurthy","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"math.OC","submitted_at":"2011-10-22T04:54:34Z","title":"Real-Time Reinforcement Learning of Constrained Markov Decision Processes with Weak Derivatives"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1110.4946","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8761e333194337f29b3678c7cea753d566fe1a23c839a14b17a09366119f2825","target":"record","created_at":"2026-05-17T23:58:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0e2864338d181dd127242090c8c408f65800a8115c8217e8de854c7330abf60d","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"math.OC","submitted_at":"2011-10-22T04:54:34Z","title_canon_sha256":"826ec5374fa49c873b4093118585e38c029a1bf145458c4b84864996d16696a2"},"schema_version":"1.0","source":{"id":"1110.4946","kind":"arxiv","version":3}},"canonical_sha256":"505953b72841590c4495e08ea27f2da73663e550fcd8fb8dcf50c87ad639af8b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"505953b72841590c4495e08ea27f2da73663e550fcd8fb8dcf50c87ad639af8b","first_computed_at":"2026-05-17T23:58:15.522664Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:58:15.522664Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"n0haXmNG5wE/AZjONFiqmbviGjJx7mP8XOc2Uha2xyiJIQiuGJs4YxVpEAi3cwEupZb4IbeyAotZOFSS+Uu4BQ==","signature_status":"signed_v1","signed_at":"2026-05-17T23:58:15.523194Z","signed_message":"canonical_sha256_bytes"},"source_id":"1110.4946","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8761e333194337f29b3678c7cea753d566fe1a23c839a14b17a09366119f2825","sha256:a78971179406896160b42308c35155ea18a4dc97d252746787763bd34f72e5a5"],"state_sha256":"e177f02675a38920902ea31d9ead3e3eaf28bc49619898f710401b170b9de655"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nCDItz+5MIn2aIqdIzl/zZhIVQy7X6wVSI5OjcfjRemKzvUBM9gjqTqYZZVvKN2LoRHlTPqO2d0+W6XVqAMFDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-23T05:08:52.318651Z","bundle_sha256":"d22b317e3be3ee7a0cec25f10b9fbbe4c5fb8bcf258c1d737f162a00ff4dcbc2"}}