{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:UZ3RM6UFOSY5DQYI5FJIZC7MDE","short_pith_number":"pith:UZ3RM6UF","canonical_record":{"source":{"id":"2107.00591","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.RO","submitted_at":"2021-07-01T16:26:54Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"810e474b1d75f10c00c494a56f69fdfe88f2db023f54213fa78d217553367c7f","abstract_canon_sha256":"9aa85167df0e285f88b15dc0dc0f09d7a0b0dd3fe8b78f4fb1c05c4f65b1d673"},"schema_version":"1.0"},"canonical_sha256":"a677167a8574b1d1c308e9528c8bec1905190c90f4dfc5abad110bd6d5f0e503","source":{"kind":"arxiv","id":"2107.00591","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.00591","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"arxiv_version","alias_value":"2107.00591v2","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.00591","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_12","alias_value":"UZ3RM6UFOSY5","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_16","alias_value":"UZ3RM6UFOSY5DQYI","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_8","alias_value":"UZ3RM6UF","created_at":"2026-07-05T03:27:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:UZ3RM6UFOSY5DQYI5FJIZC7MDE","target":"record","payload":{"canonical_record":{"source":{"id":"2107.00591","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.RO","submitted_at":"2021-07-01T16:26:54Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"810e474b1d75f10c00c494a56f69fdfe88f2db023f54213fa78d217553367c7f","abstract_canon_sha256":"9aa85167df0e285f88b15dc0dc0f09d7a0b0dd3fe8b78f4fb1c05c4f65b1d673"},"schema_version":"1.0"},"canonical_sha256":"a677167a8574b1d1c308e9528c8bec1905190c90f4dfc5abad110bd6d5f0e503","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:27:33.395115Z","signature_b64":"azNtEtYcT2AUCWCcrr7PFv9oPm0Scn5NxchRyZeyMtyR102kjsLXTBIQ2ydlC5nJBiF2uefe0KViHqqOtc//DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a677167a8574b1d1c308e9528c8bec1905190c90f4dfc5abad110bd6d5f0e503","last_reissued_at":"2026-07-05T03:27:33.394629Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:27:33.394629Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.00591","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:27:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JqFU/orhlDz/hGwJLNHzp/4LItifTq73g5Wa+wIMkMQrvUjPyi2ech35f7RgRLl1KItndfrOjPGxibQW7S54Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T07:55:22.476941Z"},"content_sha256":"982c85850e6e54b14b1802f2d975a76b647e77b16cc7d0fdd24801c3e4592c96","schema_version":"1.0","event_id":"sha256:982c85850e6e54b14b1802f2d975a76b647e77b16cc7d0fdd24801c3e4592c96"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:UZ3RM6UFOSY5DQYI5FJIZC7MDE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Offline-to-Online Reinforcement Learning via Balanced Replay and Pessimistic Q-Ensemble","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.RO","authors_text":"Jinwoo Shin, Kimin Lee, Pieter Abbeel, Seunghyun Lee, Younggyo Seo","submitted_at":"2021-07-01T16:26:54Z","abstract_excerpt":"Recent advance in deep offline reinforcement learning (RL) has made it possible to train strong robotic agents from offline datasets. However, depending on the quality of the trained agents and the application being considered, it is often desirable to fine-tune such agents via further online interactions. In this paper, we observe that state-action distribution shift may lead to severe bootstrap error during fine-tuning, which destroys the good initial policy obtained via offline RL. To address this issue, we first propose a balanced replay scheme that prioritizes samples encountered online w"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.00591","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.00591/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:27:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FpJyCWBvpV9yD0eOIZzUJI1MInunIylStS/AqsnUmi8g3IMA2//YbcRn0IgiDyIkPYbDE35gqGr/ODdfCujjBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T07:55:22.477465Z"},"content_sha256":"38472b2d775d0259865fb18a7725b64b962c8f1366764fcbef7463136a7de87b","schema_version":"1.0","event_id":"sha256:38472b2d775d0259865fb18a7725b64b962c8f1366764fcbef7463136a7de87b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/bundle.json","state_url":"https://pith.science/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T07:55:22Z","links":{"resolver":"https://pith.science/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE","bundle":"https://pith.science/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/bundle.json","state":"https://pith.science/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UZ3RM6UFOSY5DQYI5FJIZC7MDE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:UZ3RM6UFOSY5DQYI5FJIZC7MDE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9aa85167df0e285f88b15dc0dc0f09d7a0b0dd3fe8b78f4fb1c05c4f65b1d673","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.RO","submitted_at":"2021-07-01T16:26:54Z","title_canon_sha256":"810e474b1d75f10c00c494a56f69fdfe88f2db023f54213fa78d217553367c7f"},"schema_version":"1.0","source":{"id":"2107.00591","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.00591","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"arxiv_version","alias_value":"2107.00591v2","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.00591","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_12","alias_value":"UZ3RM6UFOSY5","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_16","alias_value":"UZ3RM6UFOSY5DQYI","created_at":"2026-07-05T03:27:33Z"},{"alias_kind":"pith_short_8","alias_value":"UZ3RM6UF","created_at":"2026-07-05T03:27:33Z"}],"graph_snapshots":[{"event_id":"sha256:38472b2d775d0259865fb18a7725b64b962c8f1366764fcbef7463136a7de87b","target":"graph","created_at":"2026-07-05T03:27:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.00591/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advance in deep offline reinforcement learning (RL) has made it possible to train strong robotic agents from offline datasets. However, depending on the quality of the trained agents and the application being considered, it is often desirable to fine-tune such agents via further online interactions. In this paper, we observe that state-action distribution shift may lead to severe bootstrap error during fine-tuning, which destroys the good initial policy obtained via offline RL. To address this issue, we first propose a balanced replay scheme that prioritizes samples encountered online w","authors_text":"Jinwoo Shin, Kimin Lee, Pieter Abbeel, Seunghyun Lee, Younggyo Seo","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.RO","submitted_at":"2021-07-01T16:26:54Z","title":"Offline-to-Online Reinforcement Learning via Balanced Replay and Pessimistic Q-Ensemble"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.00591","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:982c85850e6e54b14b1802f2d975a76b647e77b16cc7d0fdd24801c3e4592c96","target":"record","created_at":"2026-07-05T03:27:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9aa85167df0e285f88b15dc0dc0f09d7a0b0dd3fe8b78f4fb1c05c4f65b1d673","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.RO","submitted_at":"2021-07-01T16:26:54Z","title_canon_sha256":"810e474b1d75f10c00c494a56f69fdfe88f2db023f54213fa78d217553367c7f"},"schema_version":"1.0","source":{"id":"2107.00591","kind":"arxiv","version":2}},"canonical_sha256":"a677167a8574b1d1c308e9528c8bec1905190c90f4dfc5abad110bd6d5f0e503","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a677167a8574b1d1c308e9528c8bec1905190c90f4dfc5abad110bd6d5f0e503","first_computed_at":"2026-07-05T03:27:33.394629Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:27:33.394629Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"azNtEtYcT2AUCWCcrr7PFv9oPm0Scn5NxchRyZeyMtyR102kjsLXTBIQ2ydlC5nJBiF2uefe0KViHqqOtc//DA==","signature_status":"signed_v1","signed_at":"2026-07-05T03:27:33.395115Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.00591","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:982c85850e6e54b14b1802f2d975a76b647e77b16cc7d0fdd24801c3e4592c96","sha256:38472b2d775d0259865fb18a7725b64b962c8f1366764fcbef7463136a7de87b"],"state_sha256":"d6cf37b1ca04740a49f046264a17a8458b0157ec143551949f83bd5f5fb4b5ab"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AqhvOgbhZlBRH0Qs6SBO/YK9bZfRGCaPZpv3EnN9pYFBGh67W7jvUEqY3JWqTMmH/Qut83wWtNOglSHkZx4/Dw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T07:55:22.481504Z","bundle_sha256":"99b4bce6e5d1cb7223acc4d718666d4743a03e92e5b62ff4967c137d6600fc52"}}