{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:MA3NTS2ANU2YSXUBL57MLXS5A6","short_pith_number":"pith:MA3NTS2A","canonical_record":{"source":{"id":"1811.11646","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-28T16:05:21Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"792c45a111eb5749b37b23d4966fa1a1c519315576a6b7c64f33744837ebe61d","abstract_canon_sha256":"df018ec3ba3c660d96a18a9e3a554e2d0cf1684b5dddf07d02795713b59e2d91"},"schema_version":"1.0"},"canonical_sha256":"6036d9cb406d35895e815f7ec5de5d07852312cec8b594095b1c690c8cd193cd","source":{"kind":"arxiv","id":"1811.11646","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1811.11646","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"arxiv_version","alias_value":"1811.11646v1","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1811.11646","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"pith_short_12","alias_value":"MA3NTS2ANU2Y","created_at":"2026-05-18T12:32:37Z"},{"alias_kind":"pith_short_16","alias_value":"MA3NTS2ANU2YSXUB","created_at":"2026-05-18T12:32:37Z"},{"alias_kind":"pith_short_8","alias_value":"MA3NTS2A","created_at":"2026-05-18T12:32:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:MA3NTS2ANU2YSXUBL57MLXS5A6","target":"record","payload":{"canonical_record":{"source":{"id":"1811.11646","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-28T16:05:21Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"792c45a111eb5749b37b23d4966fa1a1c519315576a6b7c64f33744837ebe61d","abstract_canon_sha256":"df018ec3ba3c660d96a18a9e3a554e2d0cf1684b5dddf07d02795713b59e2d91"},"schema_version":"1.0"},"canonical_sha256":"6036d9cb406d35895e815f7ec5de5d07852312cec8b594095b1c690c8cd193cd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:59:40.430800Z","signature_b64":"+KImcT/ZvGzP9u7ZQqaxCyBeWsztvxXqw+qJUJlnQHIgKwgJdaq/czt+/6CN844RXfgwV2BuvOyIID8MSZvBCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6036d9cb406d35895e815f7ec5de5d07852312cec8b594095b1c690c8cd193cd","last_reissued_at":"2026-05-17T23:59:40.430014Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:59:40.430014Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1811.11646","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:59:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vvMyYHiOAjWxvloUU9aDwQJLvueP+ilMk7ttmSvkVVOUXDesaJmLa14YB/10szmMIZLxi8NU2sHKX4J9oqX+Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-26T04:29:52.452443Z"},"content_sha256":"08bc6d7af38355dd187b72945b7ad62993ee367393059d457202eaef5cca58ee","schema_version":"1.0","event_id":"sha256:08bc6d7af38355dd187b72945b7ad62993ee367393059d457202eaef5cca58ee"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:MA3NTS2ANU2YSXUBL57MLXS5A6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Structure-aware Online Learning Algorithm for Markov Decision Processes","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Abhay Karandikar, Arghyadip Roy, Prasanna Chaporkar, Vivek Borkar","submitted_at":"2018-11-28T16:05:21Z","abstract_excerpt":"To overcome the curse of dimensionality and curse of modeling in Dynamic Programming (DP) methods for solving classical Markov Decision Process (MDP) problems, Reinforcement Learning (RL) algorithms are popular. In this paper, we consider an infinite-horizon average reward MDP problem and prove the optimality of the threshold policy under certain conditions. Traditional RL techniques do not exploit the threshold nature of optimal policy while learning. In this paper, we propose a new RL algorithm which utilizes the known threshold structure of the optimal policy while learning by reducing the "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1811.11646","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:59:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oQiLGbKC6ZlEDOSlBh+vuaf6mGTAO6w2sdPxlu3tR+GUgKPfJJAFIKfY+L+6cfvJO0WehqUOjH9vSvO8foLVBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-26T04:29:52.453147Z"},"content_sha256":"9ebde2abd1033b5dcf9bb06251857f7f077ac0365185ba3bbcddf9591b119cf2","schema_version":"1.0","event_id":"sha256:9ebde2abd1033b5dcf9bb06251857f7f077ac0365185ba3bbcddf9591b119cf2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/bundle.json","state_url":"https://pith.science/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-26T04:29:52Z","links":{"resolver":"https://pith.science/pith/MA3NTS2ANU2YSXUBL57MLXS5A6","bundle":"https://pith.science/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/bundle.json","state":"https://pith.science/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MA3NTS2ANU2YSXUBL57MLXS5A6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:MA3NTS2ANU2YSXUBL57MLXS5A6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"df018ec3ba3c660d96a18a9e3a554e2d0cf1684b5dddf07d02795713b59e2d91","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-28T16:05:21Z","title_canon_sha256":"792c45a111eb5749b37b23d4966fa1a1c519315576a6b7c64f33744837ebe61d"},"schema_version":"1.0","source":{"id":"1811.11646","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1811.11646","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"arxiv_version","alias_value":"1811.11646v1","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1811.11646","created_at":"2026-05-17T23:59:40Z"},{"alias_kind":"pith_short_12","alias_value":"MA3NTS2ANU2Y","created_at":"2026-05-18T12:32:37Z"},{"alias_kind":"pith_short_16","alias_value":"MA3NTS2ANU2YSXUB","created_at":"2026-05-18T12:32:37Z"},{"alias_kind":"pith_short_8","alias_value":"MA3NTS2A","created_at":"2026-05-18T12:32:37Z"}],"graph_snapshots":[{"event_id":"sha256:9ebde2abd1033b5dcf9bb06251857f7f077ac0365185ba3bbcddf9591b119cf2","target":"graph","created_at":"2026-05-17T23:59:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"To overcome the curse of dimensionality and curse of modeling in Dynamic Programming (DP) methods for solving classical Markov Decision Process (MDP) problems, Reinforcement Learning (RL) algorithms are popular. In this paper, we consider an infinite-horizon average reward MDP problem and prove the optimality of the threshold policy under certain conditions. Traditional RL techniques do not exploit the threshold nature of optimal policy while learning. In this paper, we propose a new RL algorithm which utilizes the known threshold structure of the optimal policy while learning by reducing the ","authors_text":"Abhay Karandikar, Arghyadip Roy, Prasanna Chaporkar, Vivek Borkar","cross_cats":["stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-28T16:05:21Z","title":"A Structure-aware Online Learning Algorithm for Markov Decision Processes"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1811.11646","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:08bc6d7af38355dd187b72945b7ad62993ee367393059d457202eaef5cca58ee","target":"record","created_at":"2026-05-17T23:59:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"df018ec3ba3c660d96a18a9e3a554e2d0cf1684b5dddf07d02795713b59e2d91","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-28T16:05:21Z","title_canon_sha256":"792c45a111eb5749b37b23d4966fa1a1c519315576a6b7c64f33744837ebe61d"},"schema_version":"1.0","source":{"id":"1811.11646","kind":"arxiv","version":1}},"canonical_sha256":"6036d9cb406d35895e815f7ec5de5d07852312cec8b594095b1c690c8cd193cd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6036d9cb406d35895e815f7ec5de5d07852312cec8b594095b1c690c8cd193cd","first_computed_at":"2026-05-17T23:59:40.430014Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:59:40.430014Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+KImcT/ZvGzP9u7ZQqaxCyBeWsztvxXqw+qJUJlnQHIgKwgJdaq/czt+/6CN844RXfgwV2BuvOyIID8MSZvBCA==","signature_status":"signed_v1","signed_at":"2026-05-17T23:59:40.430800Z","signed_message":"canonical_sha256_bytes"},"source_id":"1811.11646","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:08bc6d7af38355dd187b72945b7ad62993ee367393059d457202eaef5cca58ee","sha256:9ebde2abd1033b5dcf9bb06251857f7f077ac0365185ba3bbcddf9591b119cf2"],"state_sha256":"ce14d60edd48e822119d5d567c6926b7c265c26b085471d3a22ba55b77be670f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0te6RQPNTc01RN5DjDBcJIfhtpcxIIQ1jYhvwmkYEWYSt5agcEQvORJFp2nZ/nWykAM9CDQZ+nAw9lkCZBNDDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-26T04:29:52.457108Z","bundle_sha256":"d59f2dd5f36e4219d79fc4c37ce5a468fcc0a8931e1eccbff12713fd2578a82d"}}