{"as_of":"2026-08-17T15:34:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:20cf36003e0013b8041cbb0c2f964fd5fb7b5f48c1a210bc5bcd44cc5d21bd86","coverage":[{"denominator":12,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":12,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T11:47:06.636420Z","state":"measured"},{"denominator":12,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":12,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2504.14732/citation-record","integrity":"/paper/2504.14732/integrity","json":"/paper/2504.14732/citation-record.json","paper":"/paper/2504.14732"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T11:47:06.738349Z","title":"Proof of Theorem 3 The optimistic reward function is defined as follows: R(bwn,τ ) = min R(bwn,τ ) + 4Kexp(4B) ηλmin(ΣDn) r C2 2n log 4 δ, K− 1","venue":null,"work_id":"4d46d312-80f2-43f1-ae56-1d2be97a2c40","year":2021},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.636420Z"},"links":{"citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:cff412e014ac88b387c336d32a7b6c5ab303ba6c4552edb596c5926ea7ba811b","observation_id":"7dc0ae4c-2657-497d-bf44-2b8eb12c8231","resolution":{"observed_at":"2026-08-16T11:47:06.743327Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1312.5602","last_updated":"2013-12-19T16:00:08Z","snapshot_observed_at":"2026-08-14T03:19:40.736445Z","submitted_at":"2013-12-19T16:00:08Z","title":"Playing Atari with Deep Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1312.5602","snapshot_observed_at":"2026-08-16T11:47:06.613621Z","title":"Playing atari with deep reinforcement learning.arXiv preprint arXiv:1312.5602,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.613621Z"},"links":{"cited_paper":"/paper/1312.5602","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:3ceafd9c102860a971b75c641db7039b2009f671dedd6a8403433f5718210169","observation_id":"83dd88f6-48f6-4f8b-9526-e7840fc9eb7d","resolution":{"observed_at":"2026-08-16T11:47:06.613621Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2111.04850","last_updated":"2023-02-06T10:00:56Z","snapshot_observed_at":"2026-08-16T17:43:08.040825Z","submitted_at":"2021-11-08T22:17:36Z","title":"Dueling RL: Reinforcement Learning with Trajectory Preferences","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.04850","snapshot_observed_at":"2026-08-16T11:47:06.622258Z","title":"Towards tractable optimism in model-based reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.622258Z"},"links":{"cited_paper":"/paper/2111.04850","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:b3004c70b80d70877c1258e4e6adaeaf091f280f8d7d9d7733b9624b26f90a43","observation_id":"9422b27b-8d00-417b-b05b-bc5ffe9b67d1","resolution":{"observed_at":"2026-08-16T11:47:06.622258Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2301.01392","last_updated":"2023-01-03T23:52:16Z","snapshot_observed_at":"2026-08-16T16:04:34.218992Z","submitted_at":"2023-01-03T23:52:16Z","title":"Benchmarks and Algorithms for Offline Preference-Based Reward Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.01392","snapshot_observed_at":"2026-08-16T11:47:06.625647Z","title":"Benchmarks and algorithms for offline preference-based reward learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.625647Z"},"links":{"cited_paper":"/paper/2301.01392","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:5c5c125d94899a3ccc848d89cba21f64c939d8034c024cb323e39a43b93332c2","observation_id":"fd920b3a-51b6-46b1-ba3c-441ec1baea27","resolution":{"observed_at":"2026-08-16T11:47:06.625647Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.14111","last_updated":"2023-11-03T18:36:03Z","snapshot_observed_at":"2026-08-16T15:21:23.702602Z","submitted_at":"2023-06-25T03:18:15Z","title":"Is RLHF More Difficult than Standard RL?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.14111","snapshot_observed_at":"2026-08-16T11:47:06.629390Z","title":"Is rlhf more difficult than standard rl? arXiv preprint arXiv:2306.14111,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2005,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.629390Z"},"links":{"cited_paper":"/paper/2306.14111","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:9cc5faaaab742323ed0b6ca835046a64d6b45deba271251e23ecdb3b8304b14c","observation_id":"073d7591-ac3d-45bb-bbc8-532936bd7d35","resolution":{"observed_at":"2026-08-16T11:47:06.629390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T11:47:06.759788Z","title":"Tamer: Training an agent manually via evaluative reinforcement","venue":null,"work_id":"5cbca84d-bef7-4b37-8de3-9771adaeef09","year":2008},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2013,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.607554Z"},"links":{"citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:f2ecbcf48f5542b6f8eb3135987f5ed464c17df52584d697c2cba0f757c024b5","observation_id":"b15f9e9f-7c11-4329-8018-8987f5ea50ea","resolution":{"observed_at":"2026-08-16T11:47:06.762946Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14816","last_updated":"2023-09-29T19:18:55Z","snapshot_observed_at":"2026-08-16T15:30:19.275550Z","submitted_at":"2023-05-24T07:11:26Z","title":"Provable Offline Preference-Based Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14816","snapshot_observed_at":"2026-08-16T11:47:06.632746Z","title":"Provable offline preference-based reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2016,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.632746Z"},"links":{"cited_paper":"/paper/2305.14816","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:212735394335a6c4ef9e8a51a607d75fca62c9e5c806be8da5f4f18b06131032","observation_id":"0d9849dd-3b73-4e28-9b55-105f62577960","resolution":{"observed_at":"2026-08-16T11:47:06.632746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01886","last_updated":"2024-12-13T18:59:14Z","snapshot_observed_at":"2026-08-17T07:37:46.414314Z","submitted_at":"2024-02-02T20:21:09Z","title":"Inverse Reinforcement Learning by Estimating Expertise of Demonstrators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01886","snapshot_observed_at":"2026-08-16T11:47:06.599283Z","title":"Inverse reinforcement learning by estimating expertise of demonstrators","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.599283Z"},"links":{"cited_paper":"/paper/2402.01886","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:ce201c87e0993c922a536182d22db10877e44e54bd728849c62981fce0c16edd","observation_id":"34bfff6d-8e60-41a9-b0fc-719573c10ef4","resolution":{"observed_at":"2026-08-16T11:47:06.599283Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T11:47:06.749503Z","title":"Inverse reinforcement learning with learning and leveraging demonstrators’ varying expertise levels","venue":null,"work_id":"d1b20d5d-4103-4863-a86c-7af80915aa15","year":2023},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.616791Z"},"links":{"citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:59503ce8d1900a18bff1b8dc4322006ac9aef1c87687ba546eeca998dc68acb5","observation_id":"03cfa0b1-3e01-4f43-97c0-6fbb73a6b765","resolution":{"observed_at":"2026-08-16T11:47:06.752804Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1802.06070","last_updated":"2018-10-09T23:19:52Z","snapshot_observed_at":"2026-08-14T19:45:07.867570Z","submitted_at":"2018-02-16T18:57:57Z","title":"Diversity is All You Need: Learning Skills without a Reward Function","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1802.06070","snapshot_observed_at":"2026-08-16T11:47:06.603640Z","title":"Diversity is all you need: Learning skills without a reward function","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.603640Z"},"links":{"cited_paper":"/paper/1802.06070","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:49d05c5ef14e0fea5443b7f7b88d148dea101cff63418620c5aba16a76f3cde7","observation_id":"8b0bccfa-c0d6-4eb7-8844-7b91015e5161","resolution":{"observed_at":"2026-08-16T11:47:06.603640Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2002.05138","last_updated":"2021-05-27T22:32:32Z","snapshot_observed_at":"2026-08-11T06:34:08.209616Z","submitted_at":"2020-02-12T18:30:09Z","title":"Regret Bounds for Discounted MDPs","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.05138","snapshot_observed_at":"2026-08-16T11:47:06.610498Z","title":"Regret bounds for discounted mdps","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.610498Z"},"links":{"cited_paper":"/paper/2002.05138","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:8f075ceaed5b2bdad6c37ad9d162d933a2c459f02a8d96257bd68a645eb5cad9","observation_id":"045ae104-de2c-4a14-a542-e9a172f9d9da","resolution":{"observed_at":"2026-08-16T11:47:06.610498Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1608.02732","last_updated":"2016-08-09T09:02:01Z","snapshot_observed_at":"2026-08-14T21:44:53.101212Z","submitted_at":"2016-08-09T09:02:01Z","title":"On Lower Bounds for Regret in Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1608.02732","snapshot_observed_at":"2026-08-16T11:47:06.619644Z","title":"On lower bounds for regret in reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback","version":3},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-16T11:47:06.619644Z"},"links":{"cited_paper":"/paper/1608.02732","citing_paper":"/paper/2504.14732"},"observation_digest":"sha256:caafb5aa18bcb5c16ecd55451d692e304bf0c7f95597831eb7e2e3dee972a52c","observation_id":"e4d25f93-6eda-42bf-9b74-568c7980d848","resolution":{"observed_at":"2026-08-16T11:47:06.619644Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2504.14732","last_updated":"2025-04-25T21:54:56Z","latest_version":3,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-17T08:01:25.751648Z","submitted_at":"2025-04-20T20:09:19Z","title":"Reinforcement Learning from Multi-level and Episodic Human Feedback"},"reference_resolution":{"displayed":12,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":9,"verified_exact":0,"verified_fuzzy":3},"total_outbound_references":12},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 12 of 12 outbound references and 0 inbound Pith citation observations for arXiv:2504.14732."}