{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:62P63SXFAGYI7TBYHG2PZ4UA32","short_pith_number":"pith:62P63SXF","canonical_record":{"source":{"id":"2211.06527","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-11-12T00:34:41Z","cross_cats_sorted":["cs.AI","cs.HC"],"title_canon_sha256":"81051a5f35e3a509bd8720112912f9aa744b389e313bf01f625faa900fd0412a","abstract_canon_sha256":"debb9dad72cd79a278271fa7e1f837229c7154a8ffbf210d4bfd1cadff7b779a"},"schema_version":"1.0"},"canonical_sha256":"f69fedcae501b08fcc3839b4fcf280de823840f97394ceff82eed3c69ff0774f","source":{"kind":"arxiv","id":"2211.06527","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.06527","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"arxiv_version","alias_value":"2211.06527v1","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.06527","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_12","alias_value":"62P63SXFAGYI","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_16","alias_value":"62P63SXFAGYI7TBY","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_8","alias_value":"62P63SXF","created_at":"2026-07-05T05:15:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:62P63SXFAGYI7TBYHG2PZ4UA32","target":"record","payload":{"canonical_record":{"source":{"id":"2211.06527","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-11-12T00:34:41Z","cross_cats_sorted":["cs.AI","cs.HC"],"title_canon_sha256":"81051a5f35e3a509bd8720112912f9aa744b389e313bf01f625faa900fd0412a","abstract_canon_sha256":"debb9dad72cd79a278271fa7e1f837229c7154a8ffbf210d4bfd1cadff7b779a"},"schema_version":"1.0"},"canonical_sha256":"f69fedcae501b08fcc3839b4fcf280de823840f97394ceff82eed3c69ff0774f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:15:26.077528Z","signature_b64":"axhtmYdCY13R8iFGurFoQDnP0YlCAGzkiXuTGSmKvZMkz1X23RYTt3mMZ7flUM7D12zhY10TaaQOE+paCsDwDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f69fedcae501b08fcc3839b4fcf280de823840f97394ceff82eed3c69ff0774f","last_reissued_at":"2026-07-05T05:15:26.077052Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:15:26.077052Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2211.06527","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:15:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QWt523pChpbxEA8EMAKbQ/hq8ZDaxjkUH4CL9hZL0fzXCaHnOS2UGVReuGCfilcid5/meT/2uA8KjuPvVVbGBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:43:15.063916Z"},"content_sha256":"728fbd74f62afacf7b6ed500e05ff8c9ca93ee624a203b3900c4634ac56ff7b9","schema_version":"1.0","event_id":"sha256:728fbd74f62afacf7b6ed500e05ff8c9ca93ee624a203b3900c4634ac56ff7b9"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:62P63SXFAGYI7TBYHG2PZ4UA32","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Rewards Encoding Environment Dynamics Improves Preference-based Reinforcement Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.HC"],"primary_cat":"cs.LG","authors_text":"Barry-John Theobald, Katherine Metcalf, Miguel Sarabia","submitted_at":"2022-11-12T00:34:41Z","abstract_excerpt":"Preference-based reinforcement learning (RL) algorithms help avoid the pitfalls of hand-crafted reward functions by distilling them from human preference feedback, but they remain impractical due to the burdensome number of labels required from the human, even for relatively simple tasks. In this work, we demonstrate that encoding environment dynamics in the reward function (REED) dramatically reduces the number of preference labels required in state-of-the-art preference-based RL frameworks. We hypothesize that REED-based methods better partition the state-action space and facilitate generali"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.06527","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.06527/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:15:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TmdvqbInS+7p0vVLnCXD9T4ti7HfdbfoRjE5BxEhgTpf8URj5VaybtJvvB5nCqV5adPLPtkE1G20lcS2sVTHBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:43:15.064277Z"},"content_sha256":"5e3551fbd48b8623216deeb3cb2f94cdf473ef4df648d3c9e89ae9c8da5c1a4e","schema_version":"1.0","event_id":"sha256:5e3551fbd48b8623216deeb3cb2f94cdf473ef4df648d3c9e89ae9c8da5c1a4e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/62P63SXFAGYI7TBYHG2PZ4UA32/bundle.json","state_url":"https://pith.science/pith/62P63SXFAGYI7TBYHG2PZ4UA32/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/62P63SXFAGYI7TBYHG2PZ4UA32/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T08:43:15Z","links":{"resolver":"https://pith.science/pith/62P63SXFAGYI7TBYHG2PZ4UA32","bundle":"https://pith.science/pith/62P63SXFAGYI7TBYHG2PZ4UA32/bundle.json","state":"https://pith.science/pith/62P63SXFAGYI7TBYHG2PZ4UA32/state.json","well_known_bundle":"https://pith.science/.well-known/pith/62P63SXFAGYI7TBYHG2PZ4UA32/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:62P63SXFAGYI7TBYHG2PZ4UA32","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"debb9dad72cd79a278271fa7e1f837229c7154a8ffbf210d4bfd1cadff7b779a","cross_cats_sorted":["cs.AI","cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-11-12T00:34:41Z","title_canon_sha256":"81051a5f35e3a509bd8720112912f9aa744b389e313bf01f625faa900fd0412a"},"schema_version":"1.0","source":{"id":"2211.06527","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.06527","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"arxiv_version","alias_value":"2211.06527v1","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.06527","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_12","alias_value":"62P63SXFAGYI","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_16","alias_value":"62P63SXFAGYI7TBY","created_at":"2026-07-05T05:15:26Z"},{"alias_kind":"pith_short_8","alias_value":"62P63SXF","created_at":"2026-07-05T05:15:26Z"}],"graph_snapshots":[{"event_id":"sha256:5e3551fbd48b8623216deeb3cb2f94cdf473ef4df648d3c9e89ae9c8da5c1a4e","target":"graph","created_at":"2026-07-05T05:15:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2211.06527/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Preference-based reinforcement learning (RL) algorithms help avoid the pitfalls of hand-crafted reward functions by distilling them from human preference feedback, but they remain impractical due to the burdensome number of labels required from the human, even for relatively simple tasks. In this work, we demonstrate that encoding environment dynamics in the reward function (REED) dramatically reduces the number of preference labels required in state-of-the-art preference-based RL frameworks. We hypothesize that REED-based methods better partition the state-action space and facilitate generali","authors_text":"Barry-John Theobald, Katherine Metcalf, Miguel Sarabia","cross_cats":["cs.AI","cs.HC"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-11-12T00:34:41Z","title":"Rewards Encoding Environment Dynamics Improves Preference-based Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.06527","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:728fbd74f62afacf7b6ed500e05ff8c9ca93ee624a203b3900c4634ac56ff7b9","target":"record","created_at":"2026-07-05T05:15:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"debb9dad72cd79a278271fa7e1f837229c7154a8ffbf210d4bfd1cadff7b779a","cross_cats_sorted":["cs.AI","cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-11-12T00:34:41Z","title_canon_sha256":"81051a5f35e3a509bd8720112912f9aa744b389e313bf01f625faa900fd0412a"},"schema_version":"1.0","source":{"id":"2211.06527","kind":"arxiv","version":1}},"canonical_sha256":"f69fedcae501b08fcc3839b4fcf280de823840f97394ceff82eed3c69ff0774f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f69fedcae501b08fcc3839b4fcf280de823840f97394ceff82eed3c69ff0774f","first_computed_at":"2026-07-05T05:15:26.077052Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:15:26.077052Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"axhtmYdCY13R8iFGurFoQDnP0YlCAGzkiXuTGSmKvZMkz1X23RYTt3mMZ7flUM7D12zhY10TaaQOE+paCsDwDg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:15:26.077528Z","signed_message":"canonical_sha256_bytes"},"source_id":"2211.06527","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:728fbd74f62afacf7b6ed500e05ff8c9ca93ee624a203b3900c4634ac56ff7b9","sha256:5e3551fbd48b8623216deeb3cb2f94cdf473ef4df648d3c9e89ae9c8da5c1a4e"],"state_sha256":"74aba2404fffc042ace94daf882e8ff1087e29ffa09a96a7870190ceeeb45040"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KW1GGgZJEEQl9gL72crxBV5pTNBjKlhOn4nRB9gSWiYmv0dJCQ13EbnUtKBld/QPgck4li1a+xs3W1rS+q8TAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T08:43:15.069170Z","bundle_sha256":"4710ab3cd428168023854e0e85d11b60eaceb2191ac4357f8b2e4b10e7828f6f"}}