{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:7WB6AHF3P5KFWZNL3JBCQN3SRM","short_pith_number":"pith:7WB6AHF3","canonical_record":{"source":{"id":"2205.12401","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-05-24T23:22:10Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1775a8f7dc7fe3af8be50f67aa1f392c36e311ef1096cca2eea080d5c2b13cea","abstract_canon_sha256":"741a9451fd3a4f79e310f7b536ee25a501bd3984b9cb1bc373a86dac6c021ced"},"schema_version":"1.0"},"canonical_sha256":"fd83e01cbb7f545b65abda422837728b35beff26216184b94f844dff44bcde27","source":{"kind":"arxiv","id":"2205.12401","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.12401","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"arxiv_version","alias_value":"2205.12401v1","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.12401","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_12","alias_value":"7WB6AHF3P5KF","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_16","alias_value":"7WB6AHF3P5KFWZNL","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_8","alias_value":"7WB6AHF3","created_at":"2026-07-05T04:26:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:7WB6AHF3P5KFWZNL3JBCQN3SRM","target":"record","payload":{"canonical_record":{"source":{"id":"2205.12401","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-05-24T23:22:10Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1775a8f7dc7fe3af8be50f67aa1f392c36e311ef1096cca2eea080d5c2b13cea","abstract_canon_sha256":"741a9451fd3a4f79e310f7b536ee25a501bd3984b9cb1bc373a86dac6c021ced"},"schema_version":"1.0"},"canonical_sha256":"fd83e01cbb7f545b65abda422837728b35beff26216184b94f844dff44bcde27","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:26:25.335269Z","signature_b64":"MtVd5ALqPXBkVuYyClN1nKvyZ7AndFV6YSQ2vBqpr0km2BvHCgZglp8Yv69ryAZ7mQadmWJ5Hr5Ev/Bz6MwaAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fd83e01cbb7f545b65abda422837728b35beff26216184b94f844dff44bcde27","last_reissued_at":"2026-07-05T04:26:25.334848Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:26:25.334848Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2205.12401","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:26:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aNY4VIRS31Fq506JFO3z1ME23FDxU0NtfSJliWtQAAMi/w+KfxTkZlSSf6gOOMjXvzl1MXQRl1I6iaYgcEQkBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:28:08.515811Z"},"content_sha256":"979a502079d17bd16633a82726de1bc809b8560d65b80f8e757bee4435a89c94","schema_version":"1.0","event_id":"sha256:979a502079d17bd16633a82726de1bc809b8560d65b80f8e757bee4435a89c94"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:7WB6AHF3P5KFWZNL3JBCQN3SRM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reward Uncertainty for Exploration in Preference-based Reinforcement Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Katherine Shu, Kimin Lee, Pieter Abbeel, Xinran Liang","submitted_at":"2022-05-24T23:22:10Z","abstract_excerpt":"Conveying complex objectives to reinforcement learning (RL) agents often requires meticulous reward engineering. Preference-based RL methods are able to learn a more flexible reward model based on human preferences by actively incorporating human feedback, i.e. teacher's preferences between two clips of behaviors. However, poor feedback-efficiency still remains a problem in current preference-based RL algorithms, as tailored human feedback is very expensive. To handle this issue, previous methods have mainly focused on improving query selection and policy initialization. At the same time, rece"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.12401","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2205.12401/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:26:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"x2xHkHcrijN7iZZPWI/t4nFxOrYmtiV1AGnht6atV+kgDN3BS/5o3fez1vErJbAyC7mczodR2BIsRIfbnzt4Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:28:08.516184Z"},"content_sha256":"a6e78042a575c01dd646671f64e0040708813ff7057a17da8a134bb4d7d8d9f3","schema_version":"1.0","event_id":"sha256:a6e78042a575c01dd646671f64e0040708813ff7057a17da8a134bb4d7d8d9f3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/bundle.json","state_url":"https://pith.science/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T12:28:08Z","links":{"resolver":"https://pith.science/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM","bundle":"https://pith.science/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/bundle.json","state":"https://pith.science/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7WB6AHF3P5KFWZNL3JBCQN3SRM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:7WB6AHF3P5KFWZNL3JBCQN3SRM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"741a9451fd3a4f79e310f7b536ee25a501bd3984b9cb1bc373a86dac6c021ced","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-05-24T23:22:10Z","title_canon_sha256":"1775a8f7dc7fe3af8be50f67aa1f392c36e311ef1096cca2eea080d5c2b13cea"},"schema_version":"1.0","source":{"id":"2205.12401","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.12401","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"arxiv_version","alias_value":"2205.12401v1","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.12401","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_12","alias_value":"7WB6AHF3P5KF","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_16","alias_value":"7WB6AHF3P5KFWZNL","created_at":"2026-07-05T04:26:25Z"},{"alias_kind":"pith_short_8","alias_value":"7WB6AHF3","created_at":"2026-07-05T04:26:25Z"}],"graph_snapshots":[{"event_id":"sha256:a6e78042a575c01dd646671f64e0040708813ff7057a17da8a134bb4d7d8d9f3","target":"graph","created_at":"2026-07-05T04:26:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2205.12401/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Conveying complex objectives to reinforcement learning (RL) agents often requires meticulous reward engineering. Preference-based RL methods are able to learn a more flexible reward model based on human preferences by actively incorporating human feedback, i.e. teacher's preferences between two clips of behaviors. However, poor feedback-efficiency still remains a problem in current preference-based RL algorithms, as tailored human feedback is very expensive. To handle this issue, previous methods have mainly focused on improving query selection and policy initialization. At the same time, rece","authors_text":"Katherine Shu, Kimin Lee, Pieter Abbeel, Xinran Liang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-05-24T23:22:10Z","title":"Reward Uncertainty for Exploration in Preference-based Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.12401","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:979a502079d17bd16633a82726de1bc809b8560d65b80f8e757bee4435a89c94","target":"record","created_at":"2026-07-05T04:26:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"741a9451fd3a4f79e310f7b536ee25a501bd3984b9cb1bc373a86dac6c021ced","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2022-05-24T23:22:10Z","title_canon_sha256":"1775a8f7dc7fe3af8be50f67aa1f392c36e311ef1096cca2eea080d5c2b13cea"},"schema_version":"1.0","source":{"id":"2205.12401","kind":"arxiv","version":1}},"canonical_sha256":"fd83e01cbb7f545b65abda422837728b35beff26216184b94f844dff44bcde27","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fd83e01cbb7f545b65abda422837728b35beff26216184b94f844dff44bcde27","first_computed_at":"2026-07-05T04:26:25.334848Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:26:25.334848Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MtVd5ALqPXBkVuYyClN1nKvyZ7AndFV6YSQ2vBqpr0km2BvHCgZglp8Yv69ryAZ7mQadmWJ5Hr5Ev/Bz6MwaAw==","signature_status":"signed_v1","signed_at":"2026-07-05T04:26:25.335269Z","signed_message":"canonical_sha256_bytes"},"source_id":"2205.12401","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:979a502079d17bd16633a82726de1bc809b8560d65b80f8e757bee4435a89c94","sha256:a6e78042a575c01dd646671f64e0040708813ff7057a17da8a134bb4d7d8d9f3"],"state_sha256":"90c8da9b7de850b189777f392fb3bcdd747421bed081f64aae20904d3d0cb5f4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"imo3748/O7PasQSd6KjSJSZpXUDxzt7Nf8gD8EgEZoO5e13y4qLjeK18HGEiQpnF6xj3qCLL6Q0QU913uETWCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T12:28:08.518215Z","bundle_sha256":"c5c7121d27a438aeaff66c4b23f4e52caa3a4fee91c7324f1909c3d212f79599"}}