{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:Y3FKY7FQC3KK5B7Z3FBARM3A6X","short_pith_number":"pith:Y3FKY7FQ","canonical_record":{"source":{"id":"2605.05481","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-05-06T22:02:35Z","cross_cats_sorted":[],"title_canon_sha256":"f7963600e474df518e50b85ebdc6222fd1afc458b39f64097e09183dc5d15e3b","abstract_canon_sha256":"45a1fcfab309665f999bf197fc9067f73ad0fee6e5a48de52acc5a56adb05c6c"},"schema_version":"1.0"},"canonical_sha256":"c6caac7cb016d4ae87f9d94208b360f5e2d980f058109bb59c8601ca090d6c9c","source":{"kind":"arxiv","id":"2605.05481","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.05481","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"arxiv_version","alias_value":"2605.05481v2","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.05481","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_12","alias_value":"Y3FKY7FQC3KK","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_16","alias_value":"Y3FKY7FQC3KK5B7Z","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_8","alias_value":"Y3FKY7FQ","created_at":"2026-06-19T16:12:20Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:Y3FKY7FQC3KK5B7Z3FBARM3A6X","target":"record","payload":{"canonical_record":{"source":{"id":"2605.05481","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-05-06T22:02:35Z","cross_cats_sorted":[],"title_canon_sha256":"f7963600e474df518e50b85ebdc6222fd1afc458b39f64097e09183dc5d15e3b","abstract_canon_sha256":"45a1fcfab309665f999bf197fc9067f73ad0fee6e5a48de52acc5a56adb05c6c"},"schema_version":"1.0"},"canonical_sha256":"c6caac7cb016d4ae87f9d94208b360f5e2d980f058109bb59c8601ca090d6c9c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-19T16:12:20.544782Z","signature_b64":"A9pupYAJdqNeUtmQ8Fscqw2EF4VZDPsNL5AnIy/xXz6PSAKgRKqloel05kL+ISQtgwdFCa/5jFH7YXCUdGgOCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c6caac7cb016d4ae87f9d94208b360f5e2d980f058109bb59c8601ca090d6c9c","last_reissued_at":"2026-06-19T16:12:20.544354Z","signature_status":"signed_v1","first_computed_at":"2026-06-19T16:12:20.544354Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2605.05481","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-19T16:12:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jArTGi8Gg9ByF9xa0WYjLVsnf/r7OxiYzh/NvB3+JdeFs2OZSKlHrWHakCX+UYY/9i4xNh0IhE1ODVKNYKa+AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:01:23.718979Z"},"content_sha256":"9c774e1b4c2b8256a90a6e858487653d70cf45733bbf5dfefe676a047ae37913","schema_version":"1.0","event_id":"sha256:9c774e1b4c2b8256a90a6e858487653d70cf45733bbf5dfefe676a047ae37913"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:Y3FKY7FQC3KK5B7Z3FBARM3A6X","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Approximate Next Policy Sampling: Replacing Conservative Target Policy Updates in Deep RL","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution.","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Dillon Sandhu, Ronald Parr","submitted_at":"2026-05-06T22:02:35Z","abstract_excerpt":"We revisit a classic \"chicken-and-egg\" problem in reinforcement learning: to safely improve a policy, the value function must be accurate on the state-visitation distribution of the updated policy. That distribution over states is unknown and cannot be sampled for the purposes of training the value function. Conservative updates solve this problem, but at the cost of shrinking the policy update. This paper explores an alternative solution, Approximate Next Policy Sampling (ANPS), which addresses the problem by modifying the training distribution rather than constraining the policy update. ANPS"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"If certain stability criteria are met, the update is guaranteed to be safe; otherwise, it remains no less safe than standard approximate policy iteration. Applying SV-API to PPO yields SV-PPO, which matches or improves performance on high-dimensional discrete (Atari) and continuous control benchmarks while executing substantially larger target policy updates.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the iteratively updated behavioral policy can produce a training distribution that sufficiently approximates the next policy's state-visitation distribution, and that the stability criteria can be reliably defined and checked in practice without introducing new instabilities.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Approximate Next Policy Sampling approximates the next policy's state distribution during training to enable larger safe policy updates in deep RL, demonstrated by SV-PPO matching or exceeding standard PPO on Atari and continuous control tasks.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"e5391706ee5655d44e16593d6535c2bfe1713c893ff1eaee40894e13ff8838b5"},"source":{"id":"2605.05481","kind":"arxiv","version":2},"verdict":{"id":"f462c2d6-8147-458d-9238-f81b609a1389","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-08T16:53:39.076188Z","strongest_claim":"If certain stability criteria are met, the update is guaranteed to be safe; otherwise, it remains no less safe than standard approximate policy iteration. Applying SV-API to PPO yields SV-PPO, which matches or improves performance on high-dimensional discrete (Atari) and continuous control benchmarks while executing substantially larger target policy updates.","one_line_summary":"Approximate Next Policy Sampling approximates the next policy's state distribution during training to enable larger safe policy updates in deep RL, demonstrated by SV-PPO matching or exceeding standard PPO on Atari and continuous control tasks.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the iteratively updated behavioral policy can produce a training distribution that sufficiently approximates the next policy's state-visitation distribution, and that the stability criteria can be reliably defined and checked in practice without introducing new instabilities.","pith_extraction_headline":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2605.05481/integrity.json","findings":[],"available":true,"detectors_run":[{"name":"ai_meta_artifact","ran_at":"2026-05-20T09:40:56.122526Z","status":"completed","version":"1.0.0","findings_count":0},{"name":"doi_title_agreement","ran_at":"2026-05-19T20:31:19.607367Z","status":"completed","version":"1.0.0","findings_count":0},{"name":"doi_compliance","ran_at":"2026-05-19T13:30:56.926603Z","status":"completed","version":"1.0.0","findings_count":0}],"snapshot_sha256":"d4b860ce78756389c43eeb7a66d4668983e3c2e7ce5b992fec2b9fb6fbbf161c"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"f462c2d6-8147-458d-9238-f81b609a1389"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-19T16:12:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tG6oLIZ962garL9oJ9lR9KbS78tsaxLMZ1obUK3ixtJg8Fik2MlqMz+xnLsAummg1Q6kNYxLIpKQsybj0lGTBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:01:23.719772Z"},"content_sha256":"aecde9c5abf0bb77e9f63608d046e644b72b67d050860c798e89ee965ef2f9e5","schema_version":"1.0","event_id":"sha256:aecde9c5abf0bb77e9f63608d046e644b72b67d050860c798e89ee965ef2f9e5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/bundle.json","state_url":"https://pith.science/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T19:01:23Z","links":{"resolver":"https://pith.science/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X","bundle":"https://pith.science/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/bundle.json","state":"https://pith.science/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Y3FKY7FQC3KK5B7Z3FBARM3A6X/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:Y3FKY7FQC3KK5B7Z3FBARM3A6X","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"45a1fcfab309665f999bf197fc9067f73ad0fee6e5a48de52acc5a56adb05c6c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-05-06T22:02:35Z","title_canon_sha256":"f7963600e474df518e50b85ebdc6222fd1afc458b39f64097e09183dc5d15e3b"},"schema_version":"1.0","source":{"id":"2605.05481","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.05481","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"arxiv_version","alias_value":"2605.05481v2","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.05481","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_12","alias_value":"Y3FKY7FQC3KK","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_16","alias_value":"Y3FKY7FQC3KK5B7Z","created_at":"2026-06-19T16:12:20Z"},{"alias_kind":"pith_short_8","alias_value":"Y3FKY7FQ","created_at":"2026-06-19T16:12:20Z"}],"graph_snapshots":[{"event_id":"sha256:aecde9c5abf0bb77e9f63608d046e644b72b67d050860c798e89ee965ef2f9e5","target":"graph","created_at":"2026-06-19T16:12:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"If certain stability criteria are met, the update is guaranteed to be safe; otherwise, it remains no less safe than standard approximate policy iteration. Applying SV-API to PPO yields SV-PPO, which matches or improves performance on high-dimensional discrete (Atari) and continuous control benchmarks while executing substantially larger target policy updates."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the iteratively updated behavioral policy can produce a training distribution that sufficiently approximates the next policy's state-visitation distribution, and that the stability criteria can be reliably defined and checked in practice without introducing new instabilities."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Approximate Next Policy Sampling approximates the next policy's state distribution during training to enable larger safe policy updates in deep RL, demonstrated by SV-PPO matching or exceeding standard PPO on Atari and continuous control tasks."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution."}],"snapshot_sha256":"e5391706ee5655d44e16593d6535c2bfe1713c893ff1eaee40894e13ff8838b5"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[{"findings_count":0,"name":"ai_meta_artifact","ran_at":"2026-05-20T09:40:56.122526Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_title_agreement","ran_at":"2026-05-19T20:31:19.607367Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_compliance","ran_at":"2026-05-19T13:30:56.926603Z","status":"completed","version":"1.0.0"}],"endpoint":"/pith/2605.05481/integrity.json","findings":[],"snapshot_sha256":"d4b860ce78756389c43eeb7a66d4668983e3c2e7ce5b992fec2b9fb6fbbf161c","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We revisit a classic \"chicken-and-egg\" problem in reinforcement learning: to safely improve a policy, the value function must be accurate on the state-visitation distribution of the updated policy. That distribution over states is unknown and cannot be sampled for the purposes of training the value function. Conservative updates solve this problem, but at the cost of shrinking the policy update. This paper explores an alternative solution, Approximate Next Policy Sampling (ANPS), which addresses the problem by modifying the training distribution rather than constraining the policy update. ANPS","authors_text":"Dillon Sandhu, Ronald Parr","cross_cats":[],"headline":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-05-06T22:02:35Z","title":"Approximate Next Policy Sampling: Replacing Conservative Target Policy Updates in Deep RL"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2605.05481","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-08T16:53:39.076188Z","id":"f462c2d6-8147-458d-9238-f81b609a1389","model_set":{"reader":"grok-4.3"},"one_line_summary":"Approximate Next Policy Sampling approximates the next policy's state distribution during training to enable larger safe policy updates in deep RL, demonstrated by SV-PPO matching or exceeding standard PPO on Atari and continuous control tasks.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Approximate Next Policy Sampling lets deep RL agents make larger policy updates safely by approximating the next policy's state distribution.","strongest_claim":"If certain stability criteria are met, the update is guaranteed to be safe; otherwise, it remains no less safe than standard approximate policy iteration. Applying SV-API to PPO yields SV-PPO, which matches or improves performance on high-dimensional discrete (Atari) and continuous control benchmarks while executing substantially larger target policy updates.","weakest_assumption":"That the iteratively updated behavioral policy can produce a training distribution that sufficiently approximates the next policy's state-visitation distribution, and that the stability criteria can be reliably defined and checked in practice without introducing new instabilities."}},"verdict_id":"f462c2d6-8147-458d-9238-f81b609a1389"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9c774e1b4c2b8256a90a6e858487653d70cf45733bbf5dfefe676a047ae37913","target":"record","created_at":"2026-06-19T16:12:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"45a1fcfab309665f999bf197fc9067f73ad0fee6e5a48de52acc5a56adb05c6c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2026-05-06T22:02:35Z","title_canon_sha256":"f7963600e474df518e50b85ebdc6222fd1afc458b39f64097e09183dc5d15e3b"},"schema_version":"1.0","source":{"id":"2605.05481","kind":"arxiv","version":2}},"canonical_sha256":"c6caac7cb016d4ae87f9d94208b360f5e2d980f058109bb59c8601ca090d6c9c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c6caac7cb016d4ae87f9d94208b360f5e2d980f058109bb59c8601ca090d6c9c","first_computed_at":"2026-06-19T16:12:20.544354Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-06-19T16:12:20.544354Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A9pupYAJdqNeUtmQ8Fscqw2EF4VZDPsNL5AnIy/xXz6PSAKgRKqloel05kL+ISQtgwdFCa/5jFH7YXCUdGgOCA==","signature_status":"signed_v1","signed_at":"2026-06-19T16:12:20.544782Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.05481","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9c774e1b4c2b8256a90a6e858487653d70cf45733bbf5dfefe676a047ae37913","sha256:aecde9c5abf0bb77e9f63608d046e644b72b67d050860c798e89ee965ef2f9e5"],"state_sha256":"90dc0c290c9ea64977f434aa8c78fd852e5ec406768b65a2c383a3c26151609a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EMsg6e6G/sWoEmBAX0khcdLR7kMezyLLYRXxneNeGks88ymPMtSZLsEn9lQmR8bo5Jsv3nor+zGfMbLPSOzLCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T19:01:23.725835Z","bundle_sha256":"4145d3f66d7a2af2d9fc5ed2d8449774f41f9446d4878bccdb735ca15653e1f9"}}