{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:KHA2LDXTURWT6XCP5PBAUXEPRB","short_pith_number":"pith:KHA2LDXT","canonical_record":{"source":{"id":"2303.17396","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-03-30T14:08:31Z","cross_cats_sorted":[],"title_canon_sha256":"80a4f259324c299f286c7d9981553180a1c6c7b97aff39ae5aa9f76c7c7f1f83","abstract_canon_sha256":"26de1d1061218012b7611c2a4f9cf3609696bd8fa0b06b0a7fa486c4af4187c1"},"schema_version":"1.0"},"canonical_sha256":"51c1a58ef3a46d3f5c4febc20a5c8f886eeb1aa9523534f96666c6cbf06b2c07","source":{"kind":"arxiv","id":"2303.17396","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2303.17396","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"arxiv_version","alias_value":"2303.17396v1","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2303.17396","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_12","alias_value":"KHA2LDXTURWT","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_16","alias_value":"KHA2LDXTURWT6XCP","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_8","alias_value":"KHA2LDXT","created_at":"2026-07-05T05:56:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:KHA2LDXTURWT6XCP5PBAUXEPRB","target":"record","payload":{"canonical_record":{"source":{"id":"2303.17396","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-03-30T14:08:31Z","cross_cats_sorted":[],"title_canon_sha256":"80a4f259324c299f286c7d9981553180a1c6c7b97aff39ae5aa9f76c7c7f1f83","abstract_canon_sha256":"26de1d1061218012b7611c2a4f9cf3609696bd8fa0b06b0a7fa486c4af4187c1"},"schema_version":"1.0"},"canonical_sha256":"51c1a58ef3a46d3f5c4febc20a5c8f886eeb1aa9523534f96666c6cbf06b2c07","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:56:33.710416Z","signature_b64":"rt02KzYDo7+Zm5phpd82AnvFLd8EiJ1hUyqhQBIBLb92XH8QYNZVzLznv9sRn5JZ8VkLlpF6efhEnQg//f5fDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"51c1a58ef3a46d3f5c4febc20a5c8f886eeb1aa9523534f96666c6cbf06b2c07","last_reissued_at":"2026-07-05T05:56:33.709952Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:56:33.709952Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2303.17396","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:56:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sIYCQRmaFTllTzh9D58YNSIHScCMNu3L9YPsQJea6wSwVBsM8IF33pXwDXMsM/A3SO9E2DuEN2h2I00f5M4lDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T17:01:31.225440Z"},"content_sha256":"d9e4ab6fde5d1aa82bcee26c2826d259f378613476f211178fb974fd41bb2bc3","schema_version":"1.0","event_id":"sha256:d9e4ab6fde5d1aa82bcee26c2826d259f378613476f211178fb974fd41bb2bc3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:KHA2LDXTURWT6XCP5PBAUXEPRB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Finetuning from Offline Reinforcement Learning: Challenges, Trade-offs and Practical Solutions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Edward Grefenstette, Jackie Kay, Marc Peter Deisenroth, Yicheng Luo","submitted_at":"2023-03-30T14:08:31Z","abstract_excerpt":"Offline reinforcement learning (RL) allows for the training of competent agents from offline datasets without any interaction with the environment. Online finetuning of such offline models can further improve performance. But how should we ideally finetune agents obtained from offline RL training? While offline RL algorithms can in principle be used for finetuning, in practice, their online performance improves slowly. In contrast, we show that it is possible to use standard online off-policy algorithms for faster improvement. However, we find this approach may suffer from policy collapse, whe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2303.17396","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2303.17396/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:56:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"++7EgKt6iSOImt/tD7GiKplZbh1AoP+Bj+vUVch/Yv+4ex6UHRUR1BJ5z/WoKN3A+sIoyvEXgvHoU9aO+r0/Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T17:01:31.225773Z"},"content_sha256":"ec5330fcfd7aab3b9bc98bac3758974b64d7bc8e8db8e193df4e5b83688c5cb5","schema_version":"1.0","event_id":"sha256:ec5330fcfd7aab3b9bc98bac3758974b64d7bc8e8db8e193df4e5b83688c5cb5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/bundle.json","state_url":"https://pith.science/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T17:01:31Z","links":{"resolver":"https://pith.science/pith/KHA2LDXTURWT6XCP5PBAUXEPRB","bundle":"https://pith.science/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/bundle.json","state":"https://pith.science/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KHA2LDXTURWT6XCP5PBAUXEPRB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:KHA2LDXTURWT6XCP5PBAUXEPRB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"26de1d1061218012b7611c2a4f9cf3609696bd8fa0b06b0a7fa486c4af4187c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-03-30T14:08:31Z","title_canon_sha256":"80a4f259324c299f286c7d9981553180a1c6c7b97aff39ae5aa9f76c7c7f1f83"},"schema_version":"1.0","source":{"id":"2303.17396","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2303.17396","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"arxiv_version","alias_value":"2303.17396v1","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2303.17396","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_12","alias_value":"KHA2LDXTURWT","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_16","alias_value":"KHA2LDXTURWT6XCP","created_at":"2026-07-05T05:56:33Z"},{"alias_kind":"pith_short_8","alias_value":"KHA2LDXT","created_at":"2026-07-05T05:56:33Z"}],"graph_snapshots":[{"event_id":"sha256:ec5330fcfd7aab3b9bc98bac3758974b64d7bc8e8db8e193df4e5b83688c5cb5","target":"graph","created_at":"2026-07-05T05:56:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2303.17396/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Offline reinforcement learning (RL) allows for the training of competent agents from offline datasets without any interaction with the environment. Online finetuning of such offline models can further improve performance. But how should we ideally finetune agents obtained from offline RL training? While offline RL algorithms can in principle be used for finetuning, in practice, their online performance improves slowly. In contrast, we show that it is possible to use standard online off-policy algorithms for faster improvement. However, we find this approach may suffer from policy collapse, whe","authors_text":"Edward Grefenstette, Jackie Kay, Marc Peter Deisenroth, Yicheng Luo","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-03-30T14:08:31Z","title":"Finetuning from Offline Reinforcement Learning: Challenges, Trade-offs and Practical Solutions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2303.17396","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d9e4ab6fde5d1aa82bcee26c2826d259f378613476f211178fb974fd41bb2bc3","target":"record","created_at":"2026-07-05T05:56:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"26de1d1061218012b7611c2a4f9cf3609696bd8fa0b06b0a7fa486c4af4187c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-03-30T14:08:31Z","title_canon_sha256":"80a4f259324c299f286c7d9981553180a1c6c7b97aff39ae5aa9f76c7c7f1f83"},"schema_version":"1.0","source":{"id":"2303.17396","kind":"arxiv","version":1}},"canonical_sha256":"51c1a58ef3a46d3f5c4febc20a5c8f886eeb1aa9523534f96666c6cbf06b2c07","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"51c1a58ef3a46d3f5c4febc20a5c8f886eeb1aa9523534f96666c6cbf06b2c07","first_computed_at":"2026-07-05T05:56:33.709952Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:56:33.709952Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"rt02KzYDo7+Zm5phpd82AnvFLd8EiJ1hUyqhQBIBLb92XH8QYNZVzLznv9sRn5JZ8VkLlpF6efhEnQg//f5fDw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:56:33.710416Z","signed_message":"canonical_sha256_bytes"},"source_id":"2303.17396","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d9e4ab6fde5d1aa82bcee26c2826d259f378613476f211178fb974fd41bb2bc3","sha256:ec5330fcfd7aab3b9bc98bac3758974b64d7bc8e8db8e193df4e5b83688c5cb5"],"state_sha256":"197aea59913fab2f0e57088e37c0321b7bb8f5cfa45a91885421f3d922733ce7"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e4hCgw+XL3OtUg7h/9JhdnuVWyP6F3l8+W+ohpHgklmzRwbIZhTGajt0i4hZNPGSmfhf329yH66l2NAAG0i7Dw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T17:01:31.228868Z","bundle_sha256":"7f35b81f174416b385d21d7aec7a5f7c3cc73d97a5cf1874c63caebe5bac5955"}}