{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:2NNEJUVASLHK4U5BXDBG2S52GK","short_pith_number":"pith:2NNEJUVA","canonical_record":{"source":{"id":"2305.14816","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-24T07:11:26Z","cross_cats_sorted":["math.ST","stat.ML","stat.TH"],"title_canon_sha256":"2d87974099cc3d60a9a9c9bad75d60a8cda05437bbd16167b139b5acffe8f6a0","abstract_canon_sha256":"2fe99cc35f2360d527633194be8b029b2ce60927a7f6b03acbaa9a748570c95f"},"schema_version":"1.0"},"canonical_sha256":"d35a44d2a092ceae53a1b8c26d4bba3296abe76bbe8115a819938e779d93e453","source":{"kind":"arxiv","id":"2305.14816","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14816","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14816v2","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14816","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_12","alias_value":"2NNEJUVASLHK","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_16","alias_value":"2NNEJUVASLHK4U5B","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_8","alias_value":"2NNEJUVA","created_at":"2026-07-05T06:55:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:2NNEJUVASLHK4U5BXDBG2S52GK","target":"record","payload":{"canonical_record":{"source":{"id":"2305.14816","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-24T07:11:26Z","cross_cats_sorted":["math.ST","stat.ML","stat.TH"],"title_canon_sha256":"2d87974099cc3d60a9a9c9bad75d60a8cda05437bbd16167b139b5acffe8f6a0","abstract_canon_sha256":"2fe99cc35f2360d527633194be8b029b2ce60927a7f6b03acbaa9a748570c95f"},"schema_version":"1.0"},"canonical_sha256":"d35a44d2a092ceae53a1b8c26d4bba3296abe76bbe8115a819938e779d93e453","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:55:55.181879Z","signature_b64":"UhiPZJcpnS6wtrMWuHcSVwRP1QvkAn9W5j8RrRl5B00t0S5qLkllSLOKtf3DzivV31TrLGicAo8vXg+mD+bHAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d35a44d2a092ceae53a1b8c26d4bba3296abe76bbe8115a819938e779d93e453","last_reissued_at":"2026-07-05T06:55:55.181350Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:55:55.181350Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.14816","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:55:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8q/oEDH+FTmGO9vTAeMSYwTygDI68dsHF/lSO3RNhUzVCZeQL3u0HmC2RlqPTOP14Vk9aLYSyWtZ2QLw4nMFDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T21:00:35.285183Z"},"content_sha256":"532683230c9c82ae78637d36911c5c5eb7b422da52f6c486e2f5b107aae6c43c","schema_version":"1.0","event_id":"sha256:532683230c9c82ae78637d36911c5c5eb7b422da52f6c486e2f5b107aae6c43c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:2NNEJUVASLHK4U5BXDBG2S52GK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Provable Offline Preference-Based Reinforcement Learning","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["math.ST","stat.ML","stat.TH"],"primary_cat":"cs.LG","authors_text":"Jason D. Lee, Masatoshi Uehara, Nathan Kallus, Wenhao Zhan, Wen Sun","submitted_at":"2023-05-24T07:11:26Z","abstract_excerpt":"In this paper, we investigate the problem of offline Preference-based Reinforcement Learning (PbRL) with human feedback where feedback is available in the form of preference between trajectory pairs rather than explicit rewards. Our proposed algorithm consists of two main steps: (1) estimate the implicit reward using Maximum Likelihood Estimation (MLE) with general function approximation from offline data and (2) solve a distributionally robust planning problem over a confidence set around the MLE. We consider the general reward setting where the reward can be defined over the whole trajectory"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14816","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.14816/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:55:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"b4JShEnPOZq/pBeKRk3dxRCF0eiYPt2xPuBq+QFdLvuxvkhyj6US7H8e6zBhu/9cIU7WeiiZBEi/jYyZZ7+4DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T21:00:35.285746Z"},"content_sha256":"f2d9ef58279149fd0052c2272aabc2cdd5ccd13ca8f506c1bacf581e1e7e2e3d","schema_version":"1.0","event_id":"sha256:f2d9ef58279149fd0052c2272aabc2cdd5ccd13ca8f506c1bacf581e1e7e2e3d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2NNEJUVASLHK4U5BXDBG2S52GK/bundle.json","state_url":"https://pith.science/pith/2NNEJUVASLHK4U5BXDBG2S52GK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2NNEJUVASLHK4U5BXDBG2S52GK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T21:00:35Z","links":{"resolver":"https://pith.science/pith/2NNEJUVASLHK4U5BXDBG2S52GK","bundle":"https://pith.science/pith/2NNEJUVASLHK4U5BXDBG2S52GK/bundle.json","state":"https://pith.science/pith/2NNEJUVASLHK4U5BXDBG2S52GK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2NNEJUVASLHK4U5BXDBG2S52GK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:2NNEJUVASLHK4U5BXDBG2S52GK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2fe99cc35f2360d527633194be8b029b2ce60927a7f6b03acbaa9a748570c95f","cross_cats_sorted":["math.ST","stat.ML","stat.TH"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-24T07:11:26Z","title_canon_sha256":"2d87974099cc3d60a9a9c9bad75d60a8cda05437bbd16167b139b5acffe8f6a0"},"schema_version":"1.0","source":{"id":"2305.14816","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14816","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14816v2","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14816","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_12","alias_value":"2NNEJUVASLHK","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_16","alias_value":"2NNEJUVASLHK4U5B","created_at":"2026-07-05T06:55:55Z"},{"alias_kind":"pith_short_8","alias_value":"2NNEJUVA","created_at":"2026-07-05T06:55:55Z"}],"graph_snapshots":[{"event_id":"sha256:f2d9ef58279149fd0052c2272aabc2cdd5ccd13ca8f506c1bacf581e1e7e2e3d","target":"graph","created_at":"2026-07-05T06:55:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.14816/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we investigate the problem of offline Preference-based Reinforcement Learning (PbRL) with human feedback where feedback is available in the form of preference between trajectory pairs rather than explicit rewards. Our proposed algorithm consists of two main steps: (1) estimate the implicit reward using Maximum Likelihood Estimation (MLE) with general function approximation from offline data and (2) solve a distributionally robust planning problem over a confidence set around the MLE. We consider the general reward setting where the reward can be defined over the whole trajectory","authors_text":"Jason D. Lee, Masatoshi Uehara, Nathan Kallus, Wenhao Zhan, Wen Sun","cross_cats":["math.ST","stat.ML","stat.TH"],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-24T07:11:26Z","title":"Provable Offline Preference-Based Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14816","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:532683230c9c82ae78637d36911c5c5eb7b422da52f6c486e2f5b107aae6c43c","target":"record","created_at":"2026-07-05T06:55:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2fe99cc35f2360d527633194be8b029b2ce60927a7f6b03acbaa9a748570c95f","cross_cats_sorted":["math.ST","stat.ML","stat.TH"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-24T07:11:26Z","title_canon_sha256":"2d87974099cc3d60a9a9c9bad75d60a8cda05437bbd16167b139b5acffe8f6a0"},"schema_version":"1.0","source":{"id":"2305.14816","kind":"arxiv","version":2}},"canonical_sha256":"d35a44d2a092ceae53a1b8c26d4bba3296abe76bbe8115a819938e779d93e453","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d35a44d2a092ceae53a1b8c26d4bba3296abe76bbe8115a819938e779d93e453","first_computed_at":"2026-07-05T06:55:55.181350Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:55:55.181350Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UhiPZJcpnS6wtrMWuHcSVwRP1QvkAn9W5j8RrRl5B00t0S5qLkllSLOKtf3DzivV31TrLGicAo8vXg+mD+bHAA==","signature_status":"signed_v1","signed_at":"2026-07-05T06:55:55.181879Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.14816","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:532683230c9c82ae78637d36911c5c5eb7b422da52f6c486e2f5b107aae6c43c","sha256:f2d9ef58279149fd0052c2272aabc2cdd5ccd13ca8f506c1bacf581e1e7e2e3d"],"state_sha256":"07bbffedb86edb3a46bde68df292547db0b03bd79639910190310902f22e7630"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TfYT7BbEvUv/+nrEcWIi2ePP5CWYloom6KI+ko35a+MnMfzC0zH4aUNDlpaV9n5OI/Q0jxio1R828QCaqp07Cg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T21:00:35.289941Z","bundle_sha256":"60843d67da3ccaf48ad634e5177465662d7b803f44438314d62192d522620eed"}}