{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:7QGOOPILQEJP6XDDSS3JM5T2TE","short_pith_number":"pith:7QGOOPIL","canonical_record":{"source":{"id":"2212.14405","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-12-29T18:25:01Z","cross_cats_sorted":[],"title_canon_sha256":"6192c169e4b7598c90a96df457b170ef10148e549cdc15a1bb20e64ee8ab555a","abstract_canon_sha256":"72ec8f62cd83b17f555955dc3adc3190985fae8ad3214a3103915f3e323e9fce"},"schema_version":"1.0"},"canonical_sha256":"fc0ce73d0b8112ff5c6394b696767a993348cebb3eff6528d72a9b8d8f3f9d1f","source":{"kind":"arxiv","id":"2212.14405","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.14405","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"arxiv_version","alias_value":"2212.14405v1","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.14405","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_12","alias_value":"7QGOOPILQEJP","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_16","alias_value":"7QGOOPILQEJP6XDD","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_8","alias_value":"7QGOOPIL","created_at":"2026-07-05T05:29:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:7QGOOPILQEJP6XDDSS3JM5T2TE","target":"record","payload":{"canonical_record":{"source":{"id":"2212.14405","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-12-29T18:25:01Z","cross_cats_sorted":[],"title_canon_sha256":"6192c169e4b7598c90a96df457b170ef10148e549cdc15a1bb20e64ee8ab555a","abstract_canon_sha256":"72ec8f62cd83b17f555955dc3adc3190985fae8ad3214a3103915f3e323e9fce"},"schema_version":"1.0"},"canonical_sha256":"fc0ce73d0b8112ff5c6394b696767a993348cebb3eff6528d72a9b8d8f3f9d1f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:29:09.759917Z","signature_b64":"Dr3lbkNIsNtMLCeS9KKSNXVCI9uLtqf2TlYWBWrXK1jZVKNGPvGoWWwMbJaP9GmzDwLVUUOaPhhQnKdVVc1sDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fc0ce73d0b8112ff5c6394b696767a993348cebb3eff6528d72a9b8d8f3f9d1f","last_reissued_at":"2026-07-05T05:29:09.759405Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:29:09.759405Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2212.14405","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:29:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XvxfHNDCb0mP6K852n44brj0Ume+v4N06NJUl/Uak/sCwGXmgcNNNzhfQCE67JxMLIq71uMiUg3fapJisrZFAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T05:53:59.035882Z"},"content_sha256":"5bd57d2449ada100b7895716ba5fb88e49dd035577b9a739ec4927c65485ae98","schema_version":"1.0","event_id":"sha256:5bd57d2449ada100b7895716ba5fb88e49dd035577b9a739ec4927c65485ae98"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:7QGOOPILQEJP6XDDSS3JM5T2TE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Offline Policy Optimization in RL with Variance Regularizaton","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Animesh Garg, Doina Precup, Homanga Bharadhwaj, Lihong Li, Riashat Islam, Samarth Sinha, Samin Yeasar Arnob, Zhaoran Wang, Zhuoran Yang","submitted_at":"2022-12-29T18:25:01Z","abstract_excerpt":"Learning policies from fixed offline datasets is a key challenge to scale up reinforcement learning (RL) algorithms towards practical applications. This is often because off-policy RL algorithms suffer from distributional shift, due to mismatch between dataset and the target policy, leading to high variance and over-estimation of value functions. In this work, we propose variance regularization for offline RL algorithms, using stationary distribution corrections. We show that by using Fenchel duality, we can avoid double sampling issues for computing the gradient of the variance regularizer. T"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.14405","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.14405/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:29:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YtqU7xHQ+QUGkfa9YlHJZBQEtRZRawzPlKjCWlMAG8fTPbiAfeU8g0582tQNJKVzwlqS5k8Pu2p9kA5PfXhyBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T05:53:59.036391Z"},"content_sha256":"3538d2556eebf78cfa3ed5f2fcde8ca5b616eb098ba58ac8e403dac62dbdc830","schema_version":"1.0","event_id":"sha256:3538d2556eebf78cfa3ed5f2fcde8ca5b616eb098ba58ac8e403dac62dbdc830"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/bundle.json","state_url":"https://pith.science/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T05:53:59Z","links":{"resolver":"https://pith.science/pith/7QGOOPILQEJP6XDDSS3JM5T2TE","bundle":"https://pith.science/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/bundle.json","state":"https://pith.science/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7QGOOPILQEJP6XDDSS3JM5T2TE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:7QGOOPILQEJP6XDDSS3JM5T2TE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"72ec8f62cd83b17f555955dc3adc3190985fae8ad3214a3103915f3e323e9fce","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-12-29T18:25:01Z","title_canon_sha256":"6192c169e4b7598c90a96df457b170ef10148e549cdc15a1bb20e64ee8ab555a"},"schema_version":"1.0","source":{"id":"2212.14405","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.14405","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"arxiv_version","alias_value":"2212.14405v1","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.14405","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_12","alias_value":"7QGOOPILQEJP","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_16","alias_value":"7QGOOPILQEJP6XDD","created_at":"2026-07-05T05:29:09Z"},{"alias_kind":"pith_short_8","alias_value":"7QGOOPIL","created_at":"2026-07-05T05:29:09Z"}],"graph_snapshots":[{"event_id":"sha256:3538d2556eebf78cfa3ed5f2fcde8ca5b616eb098ba58ac8e403dac62dbdc830","target":"graph","created_at":"2026-07-05T05:29:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2212.14405/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Learning policies from fixed offline datasets is a key challenge to scale up reinforcement learning (RL) algorithms towards practical applications. This is often because off-policy RL algorithms suffer from distributional shift, due to mismatch between dataset and the target policy, leading to high variance and over-estimation of value functions. In this work, we propose variance regularization for offline RL algorithms, using stationary distribution corrections. We show that by using Fenchel duality, we can avoid double sampling issues for computing the gradient of the variance regularizer. T","authors_text":"Animesh Garg, Doina Precup, Homanga Bharadhwaj, Lihong Li, Riashat Islam, Samarth Sinha, Samin Yeasar Arnob, Zhaoran Wang, Zhuoran Yang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-12-29T18:25:01Z","title":"Offline Policy Optimization in RL with Variance Regularizaton"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.14405","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5bd57d2449ada100b7895716ba5fb88e49dd035577b9a739ec4927c65485ae98","target":"record","created_at":"2026-07-05T05:29:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"72ec8f62cd83b17f555955dc3adc3190985fae8ad3214a3103915f3e323e9fce","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-12-29T18:25:01Z","title_canon_sha256":"6192c169e4b7598c90a96df457b170ef10148e549cdc15a1bb20e64ee8ab555a"},"schema_version":"1.0","source":{"id":"2212.14405","kind":"arxiv","version":1}},"canonical_sha256":"fc0ce73d0b8112ff5c6394b696767a993348cebb3eff6528d72a9b8d8f3f9d1f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fc0ce73d0b8112ff5c6394b696767a993348cebb3eff6528d72a9b8d8f3f9d1f","first_computed_at":"2026-07-05T05:29:09.759405Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:29:09.759405Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Dr3lbkNIsNtMLCeS9KKSNXVCI9uLtqf2TlYWBWrXK1jZVKNGPvGoWWwMbJaP9GmzDwLVUUOaPhhQnKdVVc1sDA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:29:09.759917Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.14405","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5bd57d2449ada100b7895716ba5fb88e49dd035577b9a739ec4927c65485ae98","sha256:3538d2556eebf78cfa3ed5f2fcde8ca5b616eb098ba58ac8e403dac62dbdc830"],"state_sha256":"f6ea8003f33c5009d2f84b936b49a55e202f8c5373b44870103ac318d1d81367"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0jjjFGcHxw4Hz0dK4LOsNCDBNO9WHBQ+gndnNBbZsGX9th/pjD/qr7DWYmEF00qpTXSFufDWbbxMc65CYGNWBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T05:53:59.041733Z","bundle_sha256":"007dc9ac0ff82aaa3661e0550fd2be1ce3eabfe3a9287af4010a918301c2be93"}}