{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:WVH6B22IUTZS2TXKEQ3OQFYNA6","short_pith_number":"pith:WVH6B22I","canonical_record":{"source":{"id":"1709.04570","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2017-09-14T00:16:14Z","cross_cats_sorted":[],"title_canon_sha256":"eb938516485ae1d1f30be440c22de1a272d1559e1f1359833c39866f4aa7735d","abstract_canon_sha256":"c5f240409990d266347577b1cad1e97ab3b35a3730fc59b369b28e980b893f09"},"schema_version":"1.0"},"canonical_sha256":"b54fe0eb48a4f32d4eea2436e8170d079ecdde0fa5bf4217f78bbb80a7d58e3e","source":{"kind":"arxiv","id":"1709.04570","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1709.04570","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"arxiv_version","alias_value":"1709.04570v1","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1709.04570","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"pith_short_12","alias_value":"WVH6B22IUTZS","created_at":"2026-05-18T12:31:53Z"},{"alias_kind":"pith_short_16","alias_value":"WVH6B22IUTZS2TXK","created_at":"2026-05-18T12:31:53Z"},{"alias_kind":"pith_short_8","alias_value":"WVH6B22I","created_at":"2026-05-18T12:31:53Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:WVH6B22IUTZS2TXKEQ3OQFYNA6","target":"record","payload":{"canonical_record":{"source":{"id":"1709.04570","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2017-09-14T00:16:14Z","cross_cats_sorted":[],"title_canon_sha256":"eb938516485ae1d1f30be440c22de1a272d1559e1f1359833c39866f4aa7735d","abstract_canon_sha256":"c5f240409990d266347577b1cad1e97ab3b35a3730fc59b369b28e980b893f09"},"schema_version":"1.0"},"canonical_sha256":"b54fe0eb48a4f32d4eea2436e8170d079ecdde0fa5bf4217f78bbb80a7d58e3e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:35:11.265501Z","signature_b64":"XRSxSo4VcvXBf9HFSTzTNNUaFwo2oWvGDAMqf2zdbSHKxP0ncOKmojcEIi/z/urJ+xfav7jaOCglUZCLFmoNBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b54fe0eb48a4f32d4eea2436e8170d079ecdde0fa5bf4217f78bbb80a7d58e3e","last_reissued_at":"2026-05-18T00:35:11.264964Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:35:11.264964Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1709.04570","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:35:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zQ75Gb5o69q7XE/HdgWyKujDU0axsRkwU+g7zWmPZ3AF5CIPT9tzCSRwC0AuzuMb05OfHdQyfdsgxP4hO5AFCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T08:49:53.810970Z"},"content_sha256":"73f892f0d387c137eac37f4fcbf6e9d22e274f16c8037baefe81d3c89a9f2a11","schema_version":"1.0","event_id":"sha256:73f892f0d387c137eac37f4fcbf6e9d22e274f16c8037baefe81d3c89a9f2a11"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:WVH6B22IUTZS2TXKEQ3OQFYNA6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Learning Unknown Markov Decision Processes: A Thompson Sampling Approach","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Ashutosh Nayyar, Mukul Gagrani, Rahul Jain, Yi Ouyang","submitted_at":"2017-09-14T00:16:14Z","abstract_excerpt":"We consider the problem of learning an unknown Markov Decision Process (MDP) that is weakly communicating in the infinite horizon setting. We propose a Thompson Sampling-based reinforcement learning algorithm with dynamic episodes (TSDE). At the beginning of each episode, the algorithm generates a sample from the posterior distribution over the unknown model parameters. It then follows the optimal stationary policy for the sampled model for the rest of the episode. The duration of each episode is dynamically determined by two stopping criteria. The first stopping criterion controls the growth "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1709.04570","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:35:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YRMygOLxYUdahXi3LRNoAYrC6heKVBvN4Gu6sYunE6bKiIqHa/7fW5dBswrf5KyHNQsXyvBTQ8+T+drifG27DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-01T08:49:53.811331Z"},"content_sha256":"4a3f13b4d150df69baa2569113faa8656ee3b6d968cb1c87d858fbb2dbaeffa2","schema_version":"1.0","event_id":"sha256:4a3f13b4d150df69baa2569113faa8656ee3b6d968cb1c87d858fbb2dbaeffa2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/bundle.json","state_url":"https://pith.science/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-01T08:49:53Z","links":{"resolver":"https://pith.science/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6","bundle":"https://pith.science/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/bundle.json","state":"https://pith.science/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WVH6B22IUTZS2TXKEQ3OQFYNA6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:WVH6B22IUTZS2TXKEQ3OQFYNA6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c5f240409990d266347577b1cad1e97ab3b35a3730fc59b369b28e980b893f09","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2017-09-14T00:16:14Z","title_canon_sha256":"eb938516485ae1d1f30be440c22de1a272d1559e1f1359833c39866f4aa7735d"},"schema_version":"1.0","source":{"id":"1709.04570","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1709.04570","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"arxiv_version","alias_value":"1709.04570v1","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1709.04570","created_at":"2026-05-18T00:35:11Z"},{"alias_kind":"pith_short_12","alias_value":"WVH6B22IUTZS","created_at":"2026-05-18T12:31:53Z"},{"alias_kind":"pith_short_16","alias_value":"WVH6B22IUTZS2TXK","created_at":"2026-05-18T12:31:53Z"},{"alias_kind":"pith_short_8","alias_value":"WVH6B22I","created_at":"2026-05-18T12:31:53Z"}],"graph_snapshots":[{"event_id":"sha256:4a3f13b4d150df69baa2569113faa8656ee3b6d968cb1c87d858fbb2dbaeffa2","target":"graph","created_at":"2026-05-18T00:35:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"We consider the problem of learning an unknown Markov Decision Process (MDP) that is weakly communicating in the infinite horizon setting. We propose a Thompson Sampling-based reinforcement learning algorithm with dynamic episodes (TSDE). At the beginning of each episode, the algorithm generates a sample from the posterior distribution over the unknown model parameters. It then follows the optimal stationary policy for the sampled model for the rest of the episode. The duration of each episode is dynamically determined by two stopping criteria. The first stopping criterion controls the growth ","authors_text":"Ashutosh Nayyar, Mukul Gagrani, Rahul Jain, Yi Ouyang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2017-09-14T00:16:14Z","title":"Learning Unknown Markov Decision Processes: A Thompson Sampling Approach"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1709.04570","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:73f892f0d387c137eac37f4fcbf6e9d22e274f16c8037baefe81d3c89a9f2a11","target":"record","created_at":"2026-05-18T00:35:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c5f240409990d266347577b1cad1e97ab3b35a3730fc59b369b28e980b893f09","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2017-09-14T00:16:14Z","title_canon_sha256":"eb938516485ae1d1f30be440c22de1a272d1559e1f1359833c39866f4aa7735d"},"schema_version":"1.0","source":{"id":"1709.04570","kind":"arxiv","version":1}},"canonical_sha256":"b54fe0eb48a4f32d4eea2436e8170d079ecdde0fa5bf4217f78bbb80a7d58e3e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b54fe0eb48a4f32d4eea2436e8170d079ecdde0fa5bf4217f78bbb80a7d58e3e","first_computed_at":"2026-05-18T00:35:11.264964Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:35:11.264964Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XRSxSo4VcvXBf9HFSTzTNNUaFwo2oWvGDAMqf2zdbSHKxP0ncOKmojcEIi/z/urJ+xfav7jaOCglUZCLFmoNBw==","signature_status":"signed_v1","signed_at":"2026-05-18T00:35:11.265501Z","signed_message":"canonical_sha256_bytes"},"source_id":"1709.04570","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:73f892f0d387c137eac37f4fcbf6e9d22e274f16c8037baefe81d3c89a9f2a11","sha256:4a3f13b4d150df69baa2569113faa8656ee3b6d968cb1c87d858fbb2dbaeffa2"],"state_sha256":"a074b28da6fa983614544c3664169efe8c9a929e95980aafa1299081d6cd01d1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IuNEKKtag0Nxqynnw2lTRR9kw0dbIyJ3aw4POqFezPr8xSkpPGePbQPI5UO8D6JDBiZkHNdNlYn5okzaDFSjCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-01T08:49:53.813235Z","bundle_sha256":"643f28884145bbe5fbc7dc4b2da4881e0ccdfd67b777bd1b17c9660d03f55603"}}