{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:36NM57SX5D5AMEA6U2XFCMHUW6","short_pith_number":"pith:36NM57SX","canonical_record":{"source":{"id":"2310.11531","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-17T19:01:08Z","cross_cats_sorted":["cs.AI","cs.SY","eess.SY","stat.ML"],"title_canon_sha256":"fddc3aa4c6dac916eff3632d3d700c081b9743f77e5519f5e48dadb150fcbd35","abstract_canon_sha256":"233d7e400ae8b7ae86ec34626bafe7a9cd6d807b731f1eb197e330f6dfa9e9b6"},"schema_version":"1.0"},"canonical_sha256":"df9acefe57e8fa06101ea6ae5130f4b7a0d4cbab75a1895b6466be1a5e3bba28","source":{"kind":"arxiv","id":"2310.11531","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.11531","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"arxiv_version","alias_value":"2310.11531v2","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.11531","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_12","alias_value":"36NM57SX5D5A","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_16","alias_value":"36NM57SX5D5AMEA6","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_8","alias_value":"36NM57SX","created_at":"2026-07-05T07:40:23Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:36NM57SX5D5AMEA6U2XFCMHUW6","target":"record","payload":{"canonical_record":{"source":{"id":"2310.11531","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-17T19:01:08Z","cross_cats_sorted":["cs.AI","cs.SY","eess.SY","stat.ML"],"title_canon_sha256":"fddc3aa4c6dac916eff3632d3d700c081b9743f77e5519f5e48dadb150fcbd35","abstract_canon_sha256":"233d7e400ae8b7ae86ec34626bafe7a9cd6d807b731f1eb197e330f6dfa9e9b6"},"schema_version":"1.0"},"canonical_sha256":"df9acefe57e8fa06101ea6ae5130f4b7a0d4cbab75a1895b6466be1a5e3bba28","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:40:23.816263Z","signature_b64":"V+qBfu7MXHktHM0W08VsroBdpSHEpjXPI8EmwGlDtn7+fUoJ9BXlb62bj1Uja3pO/xSvhGcjoeB8a0N/TmDgDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"df9acefe57e8fa06101ea6ae5130f4b7a0d4cbab75a1895b6466be1a5e3bba28","last_reissued_at":"2026-07-05T07:40:23.815755Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:40:23.815755Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.11531","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:40:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OIUMrEZEPPfwB2QVRsuA51G7s2a8Nl9Ont6H0z5+s933UnbiLUcmaq/Bjz7pyhNhypfHvPfRoOo2Cxz7jKw0Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T08:15:51.875459Z"},"content_sha256":"70749508121c3ebbb3149eaea3388f3c0423a78b673cdaa2b7f1844ea7eda7cb","schema_version":"1.0","event_id":"sha256:70749508121c3ebbb3149eaea3388f3c0423a78b673cdaa2b7f1844ea7eda7cb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:36NM57SX5D5AMEA6U2XFCMHUW6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Efficient Online Learning with Offline Datasets for Infinite Horizon MDPs: A Bayesian Approach","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.SY","eess.SY","stat.ML"],"primary_cat":"cs.LG","authors_text":"Botao Hao, Dengwang Tang, Rahul Jain, Zheng Wen","submitted_at":"2023-10-17T19:01:08Z","abstract_excerpt":"In this paper, we study the problem of efficient online reinforcement learning in the infinite horizon setting when there is an offline dataset to start with. We assume that the offline dataset is generated by an expert but with unknown level of competence, i.e., it is not perfect and not necessarily using the optimal policy. We show that if the learning agent models the behavioral policy (parameterized by a competence parameter) used by the expert, it can do substantially better in terms of minimizing cumulative regret, than if it doesn't do that. We establish an upper bound on regret of the "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.11531","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.11531/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:40:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G+psA5pl0jmvEw4r7eFERMGMXxUl2j1kSMSerQEHfj9v/zv4VM5weA3Z1cLPMYDRl8RxeKHnIe13n5xaD3tiBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T08:15:51.875959Z"},"content_sha256":"4dede7541379c129c949a22ce6e8fd2fbb28952486b7336dfb6069eb1dfef47f","schema_version":"1.0","event_id":"sha256:4dede7541379c129c949a22ce6e8fd2fbb28952486b7336dfb6069eb1dfef47f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/36NM57SX5D5AMEA6U2XFCMHUW6/bundle.json","state_url":"https://pith.science/pith/36NM57SX5D5AMEA6U2XFCMHUW6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/36NM57SX5D5AMEA6U2XFCMHUW6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T08:15:51Z","links":{"resolver":"https://pith.science/pith/36NM57SX5D5AMEA6U2XFCMHUW6","bundle":"https://pith.science/pith/36NM57SX5D5AMEA6U2XFCMHUW6/bundle.json","state":"https://pith.science/pith/36NM57SX5D5AMEA6U2XFCMHUW6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/36NM57SX5D5AMEA6U2XFCMHUW6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:36NM57SX5D5AMEA6U2XFCMHUW6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"233d7e400ae8b7ae86ec34626bafe7a9cd6d807b731f1eb197e330f6dfa9e9b6","cross_cats_sorted":["cs.AI","cs.SY","eess.SY","stat.ML"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-17T19:01:08Z","title_canon_sha256":"fddc3aa4c6dac916eff3632d3d700c081b9743f77e5519f5e48dadb150fcbd35"},"schema_version":"1.0","source":{"id":"2310.11531","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.11531","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"arxiv_version","alias_value":"2310.11531v2","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.11531","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_12","alias_value":"36NM57SX5D5A","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_16","alias_value":"36NM57SX5D5AMEA6","created_at":"2026-07-05T07:40:23Z"},{"alias_kind":"pith_short_8","alias_value":"36NM57SX","created_at":"2026-07-05T07:40:23Z"}],"graph_snapshots":[{"event_id":"sha256:4dede7541379c129c949a22ce6e8fd2fbb28952486b7336dfb6069eb1dfef47f","target":"graph","created_at":"2026-07-05T07:40:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.11531/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we study the problem of efficient online reinforcement learning in the infinite horizon setting when there is an offline dataset to start with. We assume that the offline dataset is generated by an expert but with unknown level of competence, i.e., it is not perfect and not necessarily using the optimal policy. We show that if the learning agent models the behavioral policy (parameterized by a competence parameter) used by the expert, it can do substantially better in terms of minimizing cumulative regret, than if it doesn't do that. We establish an upper bound on regret of the ","authors_text":"Botao Hao, Dengwang Tang, Rahul Jain, Zheng Wen","cross_cats":["cs.AI","cs.SY","eess.SY","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-17T19:01:08Z","title":"Efficient Online Learning with Offline Datasets for Infinite Horizon MDPs: A Bayesian Approach"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.11531","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:70749508121c3ebbb3149eaea3388f3c0423a78b673cdaa2b7f1844ea7eda7cb","target":"record","created_at":"2026-07-05T07:40:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"233d7e400ae8b7ae86ec34626bafe7a9cd6d807b731f1eb197e330f6dfa9e9b6","cross_cats_sorted":["cs.AI","cs.SY","eess.SY","stat.ML"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-17T19:01:08Z","title_canon_sha256":"fddc3aa4c6dac916eff3632d3d700c081b9743f77e5519f5e48dadb150fcbd35"},"schema_version":"1.0","source":{"id":"2310.11531","kind":"arxiv","version":2}},"canonical_sha256":"df9acefe57e8fa06101ea6ae5130f4b7a0d4cbab75a1895b6466be1a5e3bba28","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"df9acefe57e8fa06101ea6ae5130f4b7a0d4cbab75a1895b6466be1a5e3bba28","first_computed_at":"2026-07-05T07:40:23.815755Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:40:23.815755Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"V+qBfu7MXHktHM0W08VsroBdpSHEpjXPI8EmwGlDtn7+fUoJ9BXlb62bj1Uja3pO/xSvhGcjoeB8a0N/TmDgDw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:40:23.816263Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.11531","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:70749508121c3ebbb3149eaea3388f3c0423a78b673cdaa2b7f1844ea7eda7cb","sha256:4dede7541379c129c949a22ce6e8fd2fbb28952486b7336dfb6069eb1dfef47f"],"state_sha256":"53630cd643b5ef66a6010eb6cf57ef541509457b222591eacd8d4f928225ed2a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RTsfoGYdNRyByuNJJGAhIK4OW3trKJpfk1QW5K+8XvpoZ+sjyvH3ouM9GWamCzzgIBuLIvTyNYzHDbkBHUTEDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T08:15:51.880076Z","bundle_sha256":"b45eb5fa64e3ec07028a0af2decbaa311058ea2f7dae2f36b96a6f8e897ae2f4"}}