{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:WI3IM6MLUHPDU7SA4EE7VSQV3A","short_pith_number":"pith:WI3IM6ML","canonical_record":{"source":{"id":"1911.02248","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2019-11-06T08:26:32Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"99ba689e25701ae2d404bd78d9f7c36469403a1927c34a4007c235e9cf208c4d","abstract_canon_sha256":"9ae2f55d926d748f39caf46b3fa4fd02302adc61d8b79014164e4dcd1cb41ddb"},"schema_version":"1.0"},"canonical_sha256":"b23686798ba1de3a7e40e109faca15d81f3863aaec48750468f5831a11b368c8","source":{"kind":"arxiv","id":"1911.02248","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.02248","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"arxiv_version","alias_value":"1911.02248v2","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.02248","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_12","alias_value":"WI3IM6MLUHPD","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_16","alias_value":"WI3IM6MLUHPDU7SA","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_8","alias_value":"WI3IM6ML","created_at":"2026-07-05T01:11:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:WI3IM6MLUHPDU7SA4EE7VSQV3A","target":"record","payload":{"canonical_record":{"source":{"id":"1911.02248","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2019-11-06T08:26:32Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"99ba689e25701ae2d404bd78d9f7c36469403a1927c34a4007c235e9cf208c4d","abstract_canon_sha256":"9ae2f55d926d748f39caf46b3fa4fd02302adc61d8b79014164e4dcd1cb41ddb"},"schema_version":"1.0"},"canonical_sha256":"b23686798ba1de3a7e40e109faca15d81f3863aaec48750468f5831a11b368c8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:11:10.526756Z","signature_b64":"NtqNbT8KsWV1sGvyYPo3zDXwJdDD74a/mDxDQQXn7tOp4M8E2sCM+aGYGF2+Uh4/lnJB8gEg2IxAx6tOxqstAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b23686798ba1de3a7e40e109faca15d81f3863aaec48750468f5831a11b368c8","last_reissued_at":"2026-07-05T01:11:10.526213Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:11:10.526213Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1911.02248","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:11:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"spms2NZ4RXxcjyae81SJYaTXil4Wtig9ZISC/VdGzFrpPtl79+DmoStPHYpqAUnCFU168VlvENpn6k4gE6kLAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T08:02:16.363686Z"},"content_sha256":"1c18e96df5274ab7e572b2e338296719edeb2d0c6cc6433b11d2d15b235b93d2","schema_version":"1.0","event_id":"sha256:1c18e96df5274ab7e572b2e338296719edeb2d0c6cc6433b11d2d15b235b93d2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:WI3IM6MLUHPDU7SA4EE7VSQV3A","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MBCAL: Sample Efficient and Variance Reduced Reinforcement Learning for Recommender Systems","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.IR","authors_text":"Fan Wang, Hao Tian, Lihang Liu, Xiaomin Fang, Zhiming Peng","submitted_at":"2019-11-06T08:26:32Z","abstract_excerpt":"In recommender systems such as news feed stream, it is essential to optimize the long-term utilities in the continuous user-system interaction processes. Previous works have proved the capability of reinforcement learning in this problem. However, there are many practical challenges to implement deep reinforcement learning in online systems, including low sample efficiency, uncontrollable risks, and excessive variances. To address these issues, we propose a novel reinforcement learning method, namely model-based counterfactual advantage learning (MBCAL). The proposed method takes advantage of "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.02248","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1911.02248/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:11:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LzNPQl1et4OFh7xZwcBgjwKmHCdyQp8Z16BgcbgupjSVeBI34wT8iN56PIhIRCYhfkc6T366vxJ8m8n37COEBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T08:02:16.364059Z"},"content_sha256":"58ebc6ff503fcef58e4ebfa48c2941354457eaa00dfcee1e1f7eb5cf57adac48","schema_version":"1.0","event_id":"sha256:58ebc6ff503fcef58e4ebfa48c2941354457eaa00dfcee1e1f7eb5cf57adac48"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/bundle.json","state_url":"https://pith.science/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-11T08:02:16Z","links":{"resolver":"https://pith.science/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A","bundle":"https://pith.science/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/bundle.json","state":"https://pith.science/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WI3IM6MLUHPDU7SA4EE7VSQV3A/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:WI3IM6MLUHPDU7SA4EE7VSQV3A","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9ae2f55d926d748f39caf46b3fa4fd02302adc61d8b79014164e4dcd1cb41ddb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2019-11-06T08:26:32Z","title_canon_sha256":"99ba689e25701ae2d404bd78d9f7c36469403a1927c34a4007c235e9cf208c4d"},"schema_version":"1.0","source":{"id":"1911.02248","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.02248","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"arxiv_version","alias_value":"1911.02248v2","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.02248","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_12","alias_value":"WI3IM6MLUHPD","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_16","alias_value":"WI3IM6MLUHPDU7SA","created_at":"2026-07-05T01:11:10Z"},{"alias_kind":"pith_short_8","alias_value":"WI3IM6ML","created_at":"2026-07-05T01:11:10Z"}],"graph_snapshots":[{"event_id":"sha256:58ebc6ff503fcef58e4ebfa48c2941354457eaa00dfcee1e1f7eb5cf57adac48","target":"graph","created_at":"2026-07-05T01:11:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1911.02248/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In recommender systems such as news feed stream, it is essential to optimize the long-term utilities in the continuous user-system interaction processes. Previous works have proved the capability of reinforcement learning in this problem. However, there are many practical challenges to implement deep reinforcement learning in online systems, including low sample efficiency, uncontrollable risks, and excessive variances. To address these issues, we propose a novel reinforcement learning method, namely model-based counterfactual advantage learning (MBCAL). The proposed method takes advantage of ","authors_text":"Fan Wang, Hao Tian, Lihang Liu, Xiaomin Fang, Zhiming Peng","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2019-11-06T08:26:32Z","title":"MBCAL: Sample Efficient and Variance Reduced Reinforcement Learning for Recommender Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.02248","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1c18e96df5274ab7e572b2e338296719edeb2d0c6cc6433b11d2d15b235b93d2","target":"record","created_at":"2026-07-05T01:11:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9ae2f55d926d748f39caf46b3fa4fd02302adc61d8b79014164e4dcd1cb41ddb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2019-11-06T08:26:32Z","title_canon_sha256":"99ba689e25701ae2d404bd78d9f7c36469403a1927c34a4007c235e9cf208c4d"},"schema_version":"1.0","source":{"id":"1911.02248","kind":"arxiv","version":2}},"canonical_sha256":"b23686798ba1de3a7e40e109faca15d81f3863aaec48750468f5831a11b368c8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b23686798ba1de3a7e40e109faca15d81f3863aaec48750468f5831a11b368c8","first_computed_at":"2026-07-05T01:11:10.526213Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:11:10.526213Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NtqNbT8KsWV1sGvyYPo3zDXwJdDD74a/mDxDQQXn7tOp4M8E2sCM+aGYGF2+Uh4/lnJB8gEg2IxAx6tOxqstAw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:11:10.526756Z","signed_message":"canonical_sha256_bytes"},"source_id":"1911.02248","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1c18e96df5274ab7e572b2e338296719edeb2d0c6cc6433b11d2d15b235b93d2","sha256:58ebc6ff503fcef58e4ebfa48c2941354457eaa00dfcee1e1f7eb5cf57adac48"],"state_sha256":"d7d8892417b47ae73850776ba9df3b1a494141c00bbf726a64ffe2b58595c3e7"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"To8KlJA0KhY95sMVniWo5O5lnMiEIyA0kU5/dEeJ5TVuQHCuUZu7g9hTs+gtx6jFlMFlubsVxS9XcnULt/d8Cg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-11T08:02:16.366273Z","bundle_sha256":"f359c50715eb6d7b92697be1733146249e413ce7b5598d65625f2bd9bd57dce8"}}