{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:PRWSEKXDNOW6BGQC6NYCKSUVM2","short_pith_number":"pith:PRWSEKXD","canonical_record":{"source":{"id":"2312.09187","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-12-14T18:06:17Z","cross_cats_sorted":[],"title_canon_sha256":"6b8114e76a090bc5ddcc2263971c898c948f7d11462f6ba57da048cae4fc3a45","abstract_canon_sha256":"566b04f8c8a14ff1c8a9cce053add86dcbde597fcec0f9b8fb729696068226b6"},"schema_version":"1.0"},"canonical_sha256":"7c6d222ae36bade09a02f370254a9566829532129d16d732013b22217ccc2f85","source":{"kind":"arxiv","id":"2312.09187","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.09187","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"arxiv_version","alias_value":"2312.09187v3","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.09187","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_12","alias_value":"PRWSEKXDNOW6","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_16","alias_value":"PRWSEKXDNOW6BGQC","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_8","alias_value":"PRWSEKXD","created_at":"2026-07-05T08:43:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:PRWSEKXDNOW6BGQC6NYCKSUVM2","target":"record","payload":{"canonical_record":{"source":{"id":"2312.09187","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-12-14T18:06:17Z","cross_cats_sorted":[],"title_canon_sha256":"6b8114e76a090bc5ddcc2263971c898c948f7d11462f6ba57da048cae4fc3a45","abstract_canon_sha256":"566b04f8c8a14ff1c8a9cce053add86dcbde597fcec0f9b8fb729696068226b6"},"schema_version":"1.0"},"canonical_sha256":"7c6d222ae36bade09a02f370254a9566829532129d16d732013b22217ccc2f85","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:43:24.951638Z","signature_b64":"0sm/YUACzY1ukVhr7trXgVQWTuDPA6UJV97OYnKJfX1Ndwc+7RI/ne6VjOsWVCiSL9GT0MndcMaDiExRr1OUBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7c6d222ae36bade09a02f370254a9566829532129d16d732013b22217ccc2f85","last_reissued_at":"2026-07-05T08:43:24.951134Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:43:24.951134Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.09187","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:43:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xGceKkhQal6h0Ikwga3ycTEUw4az7WS+MYTEFhEZ15ATM/H/6pY5FnI9L6NKWYFyVvWMzHNpictLjpkmjrWdBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:18:47.484496Z"},"content_sha256":"a1a793823d4a5a08d791ade431f5baf9fc9f6bcf0261daa01ae68c716d212636","schema_version":"1.0","event_id":"sha256:a1a793823d4a5a08d791ade431f5baf9fc9f6bcf0261daa01ae68c716d212636"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:PRWSEKXDNOW6BGQC6NYCKSUVM2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Vision-Language Models as a Source of Rewards","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alexander Neitz, Clare Lyle, Dan Horgan, Dmitry Nikulin, Fabio Pardo, Feryal Behbahani, Gheorghe Comanici, Harris Chan, Himanshu Sahni, Hussain Masoom, Jack Parker-Holder, John Quan, Kate Baumli, Kay McKinney, Kristian Holsheimer, Lei Zhang, Luyu Wang, Maxime Gazeau, Michael Laskin, Richie Steigerwald, Satinder Baveja, Sebastian Flennerhag, Stephen Spencer, Tim Rockt\\\"aschel, Tom Schaul, Volodymyr Mnih, Yannick Schroecker","submitted_at":"2023-12-14T18:06:17Z","abstract_excerpt":"Building generalist agents that can accomplish many goals in rich open-ended environments is one of the research frontiers for reinforcement learning. A key limiting factor for building generalist agents with RL has been the need for a large number of reward functions for achieving different goals. We investigate the feasibility of using off-the-shelf vision-language models, or VLMs, as sources of rewards for reinforcement learning agents. We show how rewards for visual achievement of a variety of language goals can be derived from the CLIP family of models, and used to train RL agents that ca"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.09187","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.09187/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:43:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9+QPLDkjDHDw+NZAQM1iNYLnnsxp4j1/LNI59Z1g0dDpKxQJ3uvFV+0aR3sXBwbJ8x4aJzMdKftpBUaZgFr7BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:18:47.484873Z"},"content_sha256":"f0eedb365b5c8927325d498fb1aecc17591ab405898caf8f8615ebe40148bdd3","schema_version":"1.0","event_id":"sha256:f0eedb365b5c8927325d498fb1aecc17591ab405898caf8f8615ebe40148bdd3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/bundle.json","state_url":"https://pith.science/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T06:18:47Z","links":{"resolver":"https://pith.science/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2","bundle":"https://pith.science/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/bundle.json","state":"https://pith.science/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PRWSEKXDNOW6BGQC6NYCKSUVM2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:PRWSEKXDNOW6BGQC6NYCKSUVM2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"566b04f8c8a14ff1c8a9cce053add86dcbde597fcec0f9b8fb729696068226b6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-12-14T18:06:17Z","title_canon_sha256":"6b8114e76a090bc5ddcc2263971c898c948f7d11462f6ba57da048cae4fc3a45"},"schema_version":"1.0","source":{"id":"2312.09187","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.09187","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"arxiv_version","alias_value":"2312.09187v3","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.09187","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_12","alias_value":"PRWSEKXDNOW6","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_16","alias_value":"PRWSEKXDNOW6BGQC","created_at":"2026-07-05T08:43:24Z"},{"alias_kind":"pith_short_8","alias_value":"PRWSEKXD","created_at":"2026-07-05T08:43:24Z"}],"graph_snapshots":[{"event_id":"sha256:f0eedb365b5c8927325d498fb1aecc17591ab405898caf8f8615ebe40148bdd3","target":"graph","created_at":"2026-07-05T08:43:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.09187/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Building generalist agents that can accomplish many goals in rich open-ended environments is one of the research frontiers for reinforcement learning. A key limiting factor for building generalist agents with RL has been the need for a large number of reward functions for achieving different goals. We investigate the feasibility of using off-the-shelf vision-language models, or VLMs, as sources of rewards for reinforcement learning agents. We show how rewards for visual achievement of a variety of language goals can be derived from the CLIP family of models, and used to train RL agents that ca","authors_text":"Alexander Neitz, Clare Lyle, Dan Horgan, Dmitry Nikulin, Fabio Pardo, Feryal Behbahani, Gheorghe Comanici, Harris Chan, Himanshu Sahni, Hussain Masoom, Jack Parker-Holder, John Quan, Kate Baumli, Kay McKinney, Kristian Holsheimer, Lei Zhang, Luyu Wang, Maxime Gazeau, Michael Laskin, Richie Steigerwald, Satinder Baveja, Sebastian Flennerhag, Stephen Spencer, Tim Rockt\\\"aschel, Tom Schaul, Volodymyr Mnih, Yannick Schroecker","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-12-14T18:06:17Z","title":"Vision-Language Models as a Source of Rewards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.09187","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a1a793823d4a5a08d791ade431f5baf9fc9f6bcf0261daa01ae68c716d212636","target":"record","created_at":"2026-07-05T08:43:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"566b04f8c8a14ff1c8a9cce053add86dcbde597fcec0f9b8fb729696068226b6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-12-14T18:06:17Z","title_canon_sha256":"6b8114e76a090bc5ddcc2263971c898c948f7d11462f6ba57da048cae4fc3a45"},"schema_version":"1.0","source":{"id":"2312.09187","kind":"arxiv","version":3}},"canonical_sha256":"7c6d222ae36bade09a02f370254a9566829532129d16d732013b22217ccc2f85","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7c6d222ae36bade09a02f370254a9566829532129d16d732013b22217ccc2f85","first_computed_at":"2026-07-05T08:43:24.951134Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:43:24.951134Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0sm/YUACzY1ukVhr7trXgVQWTuDPA6UJV97OYnKJfX1Ndwc+7RI/ne6VjOsWVCiSL9GT0MndcMaDiExRr1OUBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:43:24.951638Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.09187","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a1a793823d4a5a08d791ade431f5baf9fc9f6bcf0261daa01ae68c716d212636","sha256:f0eedb365b5c8927325d498fb1aecc17591ab405898caf8f8615ebe40148bdd3"],"state_sha256":"13e8a223efc32bdf50ea0a164d9e80435e23c104d4aa92658917ab30e0011f95"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4T6YCm6kJazkCwIb6Dby6LXsdE2KGiowufpTmofR6r1qQkUQ7QEdDwV4vkEvRzCTroKrm2hDA4tXc8sM87HFCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T06:18:47.486918Z","bundle_sha256":"3ec4626dc708a3732fbcd4084e9d55097eb6bb3a7bbeb1e87cd0b5217f63fd53"}}