{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:ROWWPTL3XVO6AO37RKJPUUGDRZ","short_pith_number":"pith:ROWWPTL3","canonical_record":{"source":{"id":"1707.05714","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SY","submitted_at":"2017-07-18T15:58:10Z","cross_cats_sorted":[],"title_canon_sha256":"36e5c4a323b5d864f4fb5295e6e9715997949da04f1e985c5ba08bfa937f8ad7","abstract_canon_sha256":"0124a7c97a9d08add61be8b78421841be222d44f5adf2db409bb66927f237adb"},"schema_version":"1.0"},"canonical_sha256":"8bad67cd7bbd5de03b7f8a92fa50c38e669cacf01b36211ddf731b068a5d128c","source":{"kind":"arxiv","id":"1707.05714","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1707.05714","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"arxiv_version","alias_value":"1707.05714v1","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1707.05714","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"pith_short_12","alias_value":"ROWWPTL3XVO6","created_at":"2026-05-18T12:31:39Z"},{"alias_kind":"pith_short_16","alias_value":"ROWWPTL3XVO6AO37","created_at":"2026-05-18T12:31:39Z"},{"alias_kind":"pith_short_8","alias_value":"ROWWPTL3","created_at":"2026-05-18T12:31:39Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:ROWWPTL3XVO6AO37RKJPUUGDRZ","target":"record","payload":{"canonical_record":{"source":{"id":"1707.05714","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SY","submitted_at":"2017-07-18T15:58:10Z","cross_cats_sorted":[],"title_canon_sha256":"36e5c4a323b5d864f4fb5295e6e9715997949da04f1e985c5ba08bfa937f8ad7","abstract_canon_sha256":"0124a7c97a9d08add61be8b78421841be222d44f5adf2db409bb66927f237adb"},"schema_version":"1.0"},"canonical_sha256":"8bad67cd7bbd5de03b7f8a92fa50c38e669cacf01b36211ddf731b068a5d128c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:40:01.643775Z","signature_b64":"cSRyb4C1Zxngn7CFg9W+C6V/rD7Y1VJv5d7TNsTrrHRgoQ3ZAEENoddRIt+CcL0x/Ob68mueRJ2NkxFbbD16AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8bad67cd7bbd5de03b7f8a92fa50c38e669cacf01b36211ddf731b068a5d128c","last_reissued_at":"2026-05-18T00:40:01.643240Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:40:01.643240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1707.05714","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:40:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"W+WyciaBl7IKl6JkS4brZB8f/EQssD7ldrdxqXv/oXCAnFicF3y/p7X81aSbJxM7MAFhEz2GHYzhYZjIulGfBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T07:01:51.474651Z"},"content_sha256":"a74a22358eee1af434f7522eb0d2cd39dd2408d3d66e63f56902c05bea18e9f0","schema_version":"1.0","event_id":"sha256:a74a22358eee1af434f7522eb0d2cd39dd2408d3d66e63f56902c05bea18e9f0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:ROWWPTL3XVO6AO37RKJPUUGDRZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Multi-Armed Bandit Approach for Online Expert Selection in Markov Decision Processes","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.SY","authors_text":"Claire Tomlin, Eric Mazumdar, Roy Dong, S. Shankar Sastry, Vicen\\c{c} R\\'ubies Royo","submitted_at":"2017-07-18T15:58:10Z","abstract_excerpt":"We formulate a multi-armed bandit (MAB) approach to choosing expert policies online in Markov decision processes (MDPs). Given a set of expert policies trained on a state and action space, the goal is to maximize the cumulative reward of our agent. The hope is to quickly find the best expert in our set. The MAB formulation allows us to quantify the performance of an algorithm in terms of the regret incurred from not choosing the best expert from the beginning. We first develop the theoretical framework for MABs in MDPs, and then present a basic regret decomposition identity. We then adapt the "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1707.05714","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:40:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aCfy4KiOtQZasTJI6MPxUAuevSYAQ+GDNb3Kcjr/fY7XYNzVrTvKBfIci6+QyKd2w16DI5UZcWZ0ygI7X1yoCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T07:01:51.475182Z"},"content_sha256":"57ac04de02800e09d1f6daf1c0a5810e671dd6c55b68bbf60054c640cd11e9e2","schema_version":"1.0","event_id":"sha256:57ac04de02800e09d1f6daf1c0a5810e671dd6c55b68bbf60054c640cd11e9e2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/bundle.json","state_url":"https://pith.science/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-27T07:01:51Z","links":{"resolver":"https://pith.science/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ","bundle":"https://pith.science/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/bundle.json","state":"https://pith.science/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ROWWPTL3XVO6AO37RKJPUUGDRZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:ROWWPTL3XVO6AO37RKJPUUGDRZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0124a7c97a9d08add61be8b78421841be222d44f5adf2db409bb66927f237adb","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SY","submitted_at":"2017-07-18T15:58:10Z","title_canon_sha256":"36e5c4a323b5d864f4fb5295e6e9715997949da04f1e985c5ba08bfa937f8ad7"},"schema_version":"1.0","source":{"id":"1707.05714","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1707.05714","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"arxiv_version","alias_value":"1707.05714v1","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1707.05714","created_at":"2026-05-18T00:40:01Z"},{"alias_kind":"pith_short_12","alias_value":"ROWWPTL3XVO6","created_at":"2026-05-18T12:31:39Z"},{"alias_kind":"pith_short_16","alias_value":"ROWWPTL3XVO6AO37","created_at":"2026-05-18T12:31:39Z"},{"alias_kind":"pith_short_8","alias_value":"ROWWPTL3","created_at":"2026-05-18T12:31:39Z"}],"graph_snapshots":[{"event_id":"sha256:57ac04de02800e09d1f6daf1c0a5810e671dd6c55b68bbf60054c640cd11e9e2","target":"graph","created_at":"2026-05-18T00:40:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"We formulate a multi-armed bandit (MAB) approach to choosing expert policies online in Markov decision processes (MDPs). Given a set of expert policies trained on a state and action space, the goal is to maximize the cumulative reward of our agent. The hope is to quickly find the best expert in our set. The MAB formulation allows us to quantify the performance of an algorithm in terms of the regret incurred from not choosing the best expert from the beginning. We first develop the theoretical framework for MABs in MDPs, and then present a basic regret decomposition identity. We then adapt the ","authors_text":"Claire Tomlin, Eric Mazumdar, Roy Dong, S. Shankar Sastry, Vicen\\c{c} R\\'ubies Royo","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SY","submitted_at":"2017-07-18T15:58:10Z","title":"A Multi-Armed Bandit Approach for Online Expert Selection in Markov Decision Processes"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1707.05714","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a74a22358eee1af434f7522eb0d2cd39dd2408d3d66e63f56902c05bea18e9f0","target":"record","created_at":"2026-05-18T00:40:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0124a7c97a9d08add61be8b78421841be222d44f5adf2db409bb66927f237adb","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SY","submitted_at":"2017-07-18T15:58:10Z","title_canon_sha256":"36e5c4a323b5d864f4fb5295e6e9715997949da04f1e985c5ba08bfa937f8ad7"},"schema_version":"1.0","source":{"id":"1707.05714","kind":"arxiv","version":1}},"canonical_sha256":"8bad67cd7bbd5de03b7f8a92fa50c38e669cacf01b36211ddf731b068a5d128c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8bad67cd7bbd5de03b7f8a92fa50c38e669cacf01b36211ddf731b068a5d128c","first_computed_at":"2026-05-18T00:40:01.643240Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:40:01.643240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cSRyb4C1Zxngn7CFg9W+C6V/rD7Y1VJv5d7TNsTrrHRgoQ3ZAEENoddRIt+CcL0x/Ob68mueRJ2NkxFbbD16AA==","signature_status":"signed_v1","signed_at":"2026-05-18T00:40:01.643775Z","signed_message":"canonical_sha256_bytes"},"source_id":"1707.05714","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a74a22358eee1af434f7522eb0d2cd39dd2408d3d66e63f56902c05bea18e9f0","sha256:57ac04de02800e09d1f6daf1c0a5810e671dd6c55b68bbf60054c640cd11e9e2"],"state_sha256":"fc3420b98d5d02d2b593698a03dfe31fc4644f4c50295d38052200460dcbe71e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"csSF/VNwPAGUh2VygTmWB6+VQzpuHcw95DjSs/THeC0mLCmLSNvRvHWw4eoFBwzoG6BnNRGV8Pq1KsiWQJsMDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-27T07:01:51.478695Z","bundle_sha256":"47e07b1a5f136bc7f8b6bccff4e6b46c0b48fafdf0869d404d8bd319f2a237d6"}}