{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:4HALXDT7R25FYZRPLNJLR3TZTU","short_pith_number":"pith:4HALXDT7","canonical_record":{"source":{"id":"2204.02246","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-04-04T12:38:58Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"278fd08f627c39ba69642cbbbc5abe7ee0e0053c8fe01bf21f450e7250b75e8f","abstract_canon_sha256":"659273c09d61c2095771a30beb3412350c86399bcc8ea87074fa1fc001baf131"},"schema_version":"1.0"},"canonical_sha256":"e1c0bb8e7f8eba5c662f5b52b8ee799d1e2d99bc2d47a668c8284c1fd337f052","source":{"kind":"arxiv","id":"2204.02246","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.02246","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"arxiv_version","alias_value":"2204.02246v3","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.02246","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_12","alias_value":"4HALXDT7R25F","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_16","alias_value":"4HALXDT7R25FYZRP","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_8","alias_value":"4HALXDT7","created_at":"2026-07-05T04:19:57Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:4HALXDT7R25FYZRPLNJLR3TZTU","target":"record","payload":{"canonical_record":{"source":{"id":"2204.02246","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-04-04T12:38:58Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"278fd08f627c39ba69642cbbbc5abe7ee0e0053c8fe01bf21f450e7250b75e8f","abstract_canon_sha256":"659273c09d61c2095771a30beb3412350c86399bcc8ea87074fa1fc001baf131"},"schema_version":"1.0"},"canonical_sha256":"e1c0bb8e7f8eba5c662f5b52b8ee799d1e2d99bc2d47a668c8284c1fd337f052","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:19:57.364309Z","signature_b64":"4996YktS0uOSSOGqYQPhPyVvXHI84sqleFAxHeOEyc6M9lt6xuM6VNMsxNdZcfU5/e89H0wJsTCP/mctAJoQAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e1c0bb8e7f8eba5c662f5b52b8ee799d1e2d99bc2d47a668c8284c1fd337f052","last_reissued_at":"2026-07-05T04:19:57.363826Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:19:57.363826Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2204.02246","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:19:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tv3a0LqR4RjIvAiMPkQXs7kPm5FP1Qquc325QejVxuxQ+Tyv8BvTVQJ/yaHjL//vermIbjTJkOJBwICmwUZlDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T00:28:08.093102Z"},"content_sha256":"e0868734ba2aac2629ebec569f5eaa16e193b417a9300dbad29ddc03bfc2edd3","schema_version":"1.0","event_id":"sha256:e0868734ba2aac2629ebec569f5eaa16e193b417a9300dbad29ddc03bfc2edd3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:4HALXDT7R25FYZRPLNJLR3TZTU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Continuously Discovering Novel Strategies via Reward-Switching Policy Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Bingliang Zhang, Wei Fu, Yi Wu, Zihan Zhou","submitted_at":"2022-04-04T12:38:58Z","abstract_excerpt":"We present Reward-Switching Policy Optimization (RSPO), a paradigm to discover diverse strategies in complex RL environments by iteratively finding novel policies that are both locally optimal and sufficiently different from existing ones. To encourage the learning policy to consistently converge towards a previously undiscovered local optimum, RSPO switches between extrinsic and intrinsic rewards via a trajectory-based novelty measurement during the optimization process. When a sampled trajectory is sufficiently distinct, RSPO performs standard policy optimization with extrinsic rewards. For "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.02246","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2204.02246/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:19:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0q8p+E13t3sT+UWY9e+03dWCm0RLmFmc2WTwDOjTCMLYufkMOBKWZ/QQlaA3rHHnrDR1oJiyKBEKQPn4q2NPDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T00:28:08.094110Z"},"content_sha256":"45f65af7f51e55f9c6ca93718931690500dd2b57e639cfe80caee225c84c22f9","schema_version":"1.0","event_id":"sha256:45f65af7f51e55f9c6ca93718931690500dd2b57e639cfe80caee225c84c22f9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4HALXDT7R25FYZRPLNJLR3TZTU/bundle.json","state_url":"https://pith.science/pith/4HALXDT7R25FYZRPLNJLR3TZTU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4HALXDT7R25FYZRPLNJLR3TZTU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-23T00:28:08Z","links":{"resolver":"https://pith.science/pith/4HALXDT7R25FYZRPLNJLR3TZTU","bundle":"https://pith.science/pith/4HALXDT7R25FYZRPLNJLR3TZTU/bundle.json","state":"https://pith.science/pith/4HALXDT7R25FYZRPLNJLR3TZTU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4HALXDT7R25FYZRPLNJLR3TZTU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:4HALXDT7R25FYZRPLNJLR3TZTU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"659273c09d61c2095771a30beb3412350c86399bcc8ea87074fa1fc001baf131","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-04-04T12:38:58Z","title_canon_sha256":"278fd08f627c39ba69642cbbbc5abe7ee0e0053c8fe01bf21f450e7250b75e8f"},"schema_version":"1.0","source":{"id":"2204.02246","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.02246","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"arxiv_version","alias_value":"2204.02246v3","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.02246","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_12","alias_value":"4HALXDT7R25F","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_16","alias_value":"4HALXDT7R25FYZRP","created_at":"2026-07-05T04:19:57Z"},{"alias_kind":"pith_short_8","alias_value":"4HALXDT7","created_at":"2026-07-05T04:19:57Z"}],"graph_snapshots":[{"event_id":"sha256:45f65af7f51e55f9c6ca93718931690500dd2b57e639cfe80caee225c84c22f9","target":"graph","created_at":"2026-07-05T04:19:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2204.02246/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present Reward-Switching Policy Optimization (RSPO), a paradigm to discover diverse strategies in complex RL environments by iteratively finding novel policies that are both locally optimal and sufficiently different from existing ones. To encourage the learning policy to consistently converge towards a previously undiscovered local optimum, RSPO switches between extrinsic and intrinsic rewards via a trajectory-based novelty measurement during the optimization process. When a sampled trajectory is sufficiently distinct, RSPO performs standard policy optimization with extrinsic rewards. For ","authors_text":"Bingliang Zhang, Wei Fu, Yi Wu, Zihan Zhou","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-04-04T12:38:58Z","title":"Continuously Discovering Novel Strategies via Reward-Switching Policy Optimization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.02246","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e0868734ba2aac2629ebec569f5eaa16e193b417a9300dbad29ddc03bfc2edd3","target":"record","created_at":"2026-07-05T04:19:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"659273c09d61c2095771a30beb3412350c86399bcc8ea87074fa1fc001baf131","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-04-04T12:38:58Z","title_canon_sha256":"278fd08f627c39ba69642cbbbc5abe7ee0e0053c8fe01bf21f450e7250b75e8f"},"schema_version":"1.0","source":{"id":"2204.02246","kind":"arxiv","version":3}},"canonical_sha256":"e1c0bb8e7f8eba5c662f5b52b8ee799d1e2d99bc2d47a668c8284c1fd337f052","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e1c0bb8e7f8eba5c662f5b52b8ee799d1e2d99bc2d47a668c8284c1fd337f052","first_computed_at":"2026-07-05T04:19:57.363826Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:19:57.363826Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4996YktS0uOSSOGqYQPhPyVvXHI84sqleFAxHeOEyc6M9lt6xuM6VNMsxNdZcfU5/e89H0wJsTCP/mctAJoQAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:19:57.364309Z","signed_message":"canonical_sha256_bytes"},"source_id":"2204.02246","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e0868734ba2aac2629ebec569f5eaa16e193b417a9300dbad29ddc03bfc2edd3","sha256:45f65af7f51e55f9c6ca93718931690500dd2b57e639cfe80caee225c84c22f9"],"state_sha256":"14a598f5438f0357033d67a0906868a1fb056cc01d9bed8b8b84e6065ab3f054"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SxrERYT2SdTJnzUa1Lx00eTzZaWTQscO11GODOdmj+IO0h26WC9PXBJMwEu4axsHT/ih9Lr1LyS1PnlQRrBBBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-23T00:28:08.099219Z","bundle_sha256":"5f26c8f67a7efd48d7958fb423e419d9e264e3085f6a33e90053e09417838da7"}}