{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:E7WIJF4IUO7RCUJRIU63TQ3NZA","short_pith_number":"pith:E7WIJF4I","canonical_record":{"source":{"id":"2505.22492","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-28T15:42:20Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"9a9784540ace959454a5b708ec4c12076a72f6c6327d13ffaad1985f6f776436","abstract_canon_sha256":"3392d1ce3bf2bc9ff827d6d17f75b392c91f57f4a1bdd220bca9a626bbff45e0"},"schema_version":"1.0"},"canonical_sha256":"27ec849788a3bf115131453db9c36dc817d3db0dbd649a703a92abb984fe039c","source":{"kind":"arxiv","id":"2505.22492","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.22492","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"arxiv_version","alias_value":"2505.22492v1","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.22492","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_12","alias_value":"E7WIJF4IUO7R","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_16","alias_value":"E7WIJF4IUO7RCUJR","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_8","alias_value":"E7WIJF4I","created_at":"2026-07-05T11:11:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:E7WIJF4IUO7RCUJRIU63TQ3NZA","target":"record","payload":{"canonical_record":{"source":{"id":"2505.22492","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-28T15:42:20Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"9a9784540ace959454a5b708ec4c12076a72f6c6327d13ffaad1985f6f776436","abstract_canon_sha256":"3392d1ce3bf2bc9ff827d6d17f75b392c91f57f4a1bdd220bca9a626bbff45e0"},"schema_version":"1.0"},"canonical_sha256":"27ec849788a3bf115131453db9c36dc817d3db0dbd649a703a92abb984fe039c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:11:25.200070Z","signature_b64":"3VZLXQKS6uA8xnLAlBXViR7Zf6BZ9R2TawPKE+Ft5FNpPUcVZsVSxeJtlx6gpexQf1tWES5XyStKWAXdUnjmDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"27ec849788a3bf115131453db9c36dc817d3db0dbd649a703a92abb984fe039c","last_reissued_at":"2026-07-05T11:11:25.199528Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:11:25.199528Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.22492","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:11:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aUbC+noWnsznqvRsaa2XwineAqSI+/gw5xrl5wWjABa3QX+25SNOKnacZuudD7dDG4hPKb+p5OPdrofcsGH6Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T09:01:01.623159Z"},"content_sha256":"d7ed58a61d4286853897155dc913924d84025c736fade72831f2a60985c096c7","schema_version":"1.0","event_id":"sha256:d7ed58a61d4286853897155dc913924d84025c736fade72831f2a60985c096c7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:E7WIJF4IUO7RCUJRIU63TQ3NZA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Demystifying the Paradox of Importance Sampling with an Estimated History-Dependent Behavior Policy in Off-Policy Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Chengchun Shi, Hongyi Zhou, Jin Zhu, Josiah P. Hanna, Ying Yang","submitted_at":"2025-05-28T15:42:20Z","abstract_excerpt":"This paper studies off-policy evaluation (OPE) in reinforcement learning with a focus on behavior policy estimation for importance sampling. Prior work has shown empirically that estimating a history-dependent behavior policy can lead to lower mean squared error (MSE) even when the true behavior policy is Markovian. However, the question of why the use of history should lower MSE remains open. In this paper, we theoretically demystify this paradox by deriving a bias-variance decomposition of the MSE of ordinary importance sampling (IS) estimators, demonstrating that history-dependent behavior "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.22492","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.22492/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:11:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RKMgmc4ln0NxpvvwtPBIGCLoO5WDgBZMmMeVIXwnljRaILsQnl0OKrVnn2KAyLLAGC3znVRtEyPtN5vsip2XBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T09:01:01.623833Z"},"content_sha256":"972b5d7a60d9ef471e1e36d77629c4c34231fab80f4a908709feb90e9a535a14","schema_version":"1.0","event_id":"sha256:972b5d7a60d9ef471e1e36d77629c4c34231fab80f4a908709feb90e9a535a14"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/bundle.json","state_url":"https://pith.science/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-19T09:01:01Z","links":{"resolver":"https://pith.science/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA","bundle":"https://pith.science/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/bundle.json","state":"https://pith.science/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/E7WIJF4IUO7RCUJRIU63TQ3NZA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:E7WIJF4IUO7RCUJRIU63TQ3NZA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3392d1ce3bf2bc9ff827d6d17f75b392c91f57f4a1bdd220bca9a626bbff45e0","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-28T15:42:20Z","title_canon_sha256":"9a9784540ace959454a5b708ec4c12076a72f6c6327d13ffaad1985f6f776436"},"schema_version":"1.0","source":{"id":"2505.22492","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.22492","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"arxiv_version","alias_value":"2505.22492v1","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.22492","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_12","alias_value":"E7WIJF4IUO7R","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_16","alias_value":"E7WIJF4IUO7RCUJR","created_at":"2026-07-05T11:11:25Z"},{"alias_kind":"pith_short_8","alias_value":"E7WIJF4I","created_at":"2026-07-05T11:11:25Z"}],"graph_snapshots":[{"event_id":"sha256:972b5d7a60d9ef471e1e36d77629c4c34231fab80f4a908709feb90e9a535a14","target":"graph","created_at":"2026-07-05T11:11:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.22492/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper studies off-policy evaluation (OPE) in reinforcement learning with a focus on behavior policy estimation for importance sampling. Prior work has shown empirically that estimating a history-dependent behavior policy can lead to lower mean squared error (MSE) even when the true behavior policy is Markovian. However, the question of why the use of history should lower MSE remains open. In this paper, we theoretically demystify this paradox by deriving a bias-variance decomposition of the MSE of ordinary importance sampling (IS) estimators, demonstrating that history-dependent behavior ","authors_text":"Chengchun Shi, Hongyi Zhou, Jin Zhu, Josiah P. Hanna, Ying Yang","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-28T15:42:20Z","title":"Demystifying the Paradox of Importance Sampling with an Estimated History-Dependent Behavior Policy in Off-Policy Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.22492","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d7ed58a61d4286853897155dc913924d84025c736fade72831f2a60985c096c7","target":"record","created_at":"2026-07-05T11:11:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3392d1ce3bf2bc9ff827d6d17f75b392c91f57f4a1bdd220bca9a626bbff45e0","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-28T15:42:20Z","title_canon_sha256":"9a9784540ace959454a5b708ec4c12076a72f6c6327d13ffaad1985f6f776436"},"schema_version":"1.0","source":{"id":"2505.22492","kind":"arxiv","version":1}},"canonical_sha256":"27ec849788a3bf115131453db9c36dc817d3db0dbd649a703a92abb984fe039c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"27ec849788a3bf115131453db9c36dc817d3db0dbd649a703a92abb984fe039c","first_computed_at":"2026-07-05T11:11:25.199528Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:11:25.199528Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3VZLXQKS6uA8xnLAlBXViR7Zf6BZ9R2TawPKE+Ft5FNpPUcVZsVSxeJtlx6gpexQf1tWES5XyStKWAXdUnjmDw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:11:25.200070Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.22492","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d7ed58a61d4286853897155dc913924d84025c736fade72831f2a60985c096c7","sha256:972b5d7a60d9ef471e1e36d77629c4c34231fab80f4a908709feb90e9a535a14"],"state_sha256":"64306b2ed4605065b02dc8504af78563c68b9ccc8b1d4f8ad8c9a170ad1554e5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TfTPaSAInFUNbOKPaNnfCqqbhq4JdLiCU3EdlLY/bgYMdxhioGOkKNG7adU7YLeoSXJGXnm+SScIPn6WkabBAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-19T09:01:01.627422Z","bundle_sha256":"d816ee87fcf231216be9f8d36c2751f10db4e0663c1a28cb2ccf140148249169"}}