{"as_of":"2026-08-08T10:13:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:a33f8562fa47178fe94e116d3b5eee58c550367ab20e791cc98b96be0619a0ab","coverage":[{"denominator":31,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":31,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T14:01:07.462011Z","state":"measured"},{"denominator":34,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":34,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-27T17:11:46.549150Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"arxiv_reference","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":0,"observed_at":"2026-08-05T02:28:24.338817Z","source":"arxiv_reference"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"cited_work":{"arxiv_id":"2505.20556","doi":"10.48550/arxiv.2505.20556","metadata_source":"arxiv_reference","pith_arxiv_id":"2505.20556","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Learning a","venue":"ArXiv.org","work_id":"45436740-3297-4649-a075-46eb1b414c71","year":null},"citing_paper":{"arxiv_id":"2605.00155","last_updated":"2026-07-09T02:28:34Z","snapshot_observed_at":"2026-07-12T23:17:30.599444Z","submitted_at":"2026-04-30T19:22:56Z","title":"Wasserstein Distributionally Robust Regret Optimization for Reinforcement Learning from Human Feedback","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-09T21:03:53.045304Z"},"links":{"cited_paper":"/paper/2505.20556","citing_paper":"/paper/2605.00155"},"observation_digest":"sha256:5bef20bf9e3ceb41bba662ec3b5726a530d0aa59afcf29525851a321f0dad0d9","observation_id":"b4b2d56e-a2c4-49c5-af8c-331bc75722da","resolution":{"observed_at":"2026-05-11T14:46:47.349064Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"cited_work":{"arxiv_id":"2505.20556","doi":"10.48550/arxiv.2505.20556","metadata_source":"arxiv_reference","pith_arxiv_id":"2505.20556","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Learning a","venue":"ArXiv.org","work_id":"45436740-3297-4649-a075-46eb1b414c71","year":null},"citing_paper":{"arxiv_id":"2605.21602","last_updated":"2026-05-24T21:06:51Z","snapshot_observed_at":"2026-07-31T05:05:49.494448Z","submitted_at":"2026-05-20T18:08:21Z","title":"Benchmarking and Improving Monitors for Out-Of-Distribution Alignment Failure in LLMs","version":1},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-05-22T09:38:04.387777Z"},"links":{"cited_paper":"/paper/2505.20556","citing_paper":"/paper/2605.21602"},"observation_digest":"sha256:a1cb0cef9ea914437fd85d9896438c481155712ebc96e728cace03472e0f2880","observation_id":"3eb09b5e-b6d2-4617-b84c-8e79bd1ac3ba","resolution":{"observed_at":"2026-05-22T09:41:21.460204Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"cited_work":{"arxiv_id":"2505.20556","doi":"10.48550/arxiv.2505.20556","metadata_source":"arxiv_reference","pith_arxiv_id":"2505.20556","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Learning a","venue":"ArXiv.org","work_id":"45436740-3297-4649-a075-46eb1b414c71","year":null},"citing_paper":{"arxiv_id":"2606.09073","last_updated":"2026-06-10T20:16:58Z","snapshot_observed_at":"2026-07-06T23:48:30.569726Z","submitted_at":"2026-06-08T06:15:30Z","title":"A Unifying Lens on Reward Uncertainty in RLHF","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-06-27T17:11:46.549150Z"},"links":{"cited_paper":"/paper/2505.20556","citing_paper":"/paper/2606.09073"},"observation_digest":"sha256:f391f5299885b27647a71cb712ca0e6a26876b8c301e2e5d64949c32c219f8a4","observation_id":"bfa36826-0ee0-4a04-925f-e4fd9f11aaef","resolution":{"observed_at":"2026-07-03T00:27:29.825574Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-22T21:23:21.018483+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2505.20556/citation-record","integrity":"/paper/2505.20556/integrity","json":"/paper/2505.20556/citation-record.json","paper":"/paper/2505.20556"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:01:09.025424Z","title":"[2024b], with probability at least 1 − δ, the last term can be bound by V µ r∗ (π) − V µ r∗ (ˆπ) ≤ CµD (R, π, πref)2 8κ2 · β + 3β N log( Nϵ(R, ∥ · ∥∞) δ )","venue":null,"work_id":"5e61ba40-f2ea-4a83-966c-428296f91992","year":2022},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:07.287615Z"},"links":{"citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:00a2c4990495ac32cc863ca2148a13ab66eb27b1edb7e247a1417b7ffbef810b","observation_id":"de6fb29f-26b0-426e-92ab-19cb013c0f0c","resolution":{"observed_at":"2026-08-07T14:01:09.146820Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:01:08.602017Z","title":"The rejection sampling process is effectively a policy as it takes a prompt as input and stochastically outputs a response","venue":null,"work_id":"994c885f-d326-4e02-b968-53fcb1476dc6","year":2024},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:07.462011Z"},"links":{"citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:24cbf0e228237a19615c61c1b3a5d5e3033e145f5a039e1bfba119db01b8f0de","observation_id":"36e8ddd2-9c26-4b78-93aa-403379b37e37","resolution":{"observed_at":"2026-08-07T14:01:08.680829Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.09244","last_updated":"2024-08-16T23:59:29Z","snapshot_observed_at":"2026-08-06T16:36:51.551387Z","submitted_at":"2023-12-14T18:59:04Z","title":"Helping or Herding? Reward Model Ensembles Mitigate but do not Eliminate Reward Hacking","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.09244","snapshot_observed_at":"2026-08-07T14:01:04.764200Z","title":"Helping or herd- ing? reward model ensembles mitigate but do not eliminate reward hacking","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.764200Z"},"links":{"cited_paper":"/paper/2312.09244","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:ae8772aaf6958a69a6cb0329127b94bb1e070717d6e467a591dc6615a71a2a05","observation_id":"85ccfc5b-4c63-4600-9e91-6f20604f8e6a","resolution":{"observed_at":"2026-08-07T14:01:04.764200Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19316","last_updated":"2025-03-03T08:22:25Z","snapshot_observed_at":"2026-08-02T12:00:09.704648Z","submitted_at":"2024-05-29T17:39:48Z","title":"Robust Preference Optimization through Reward Model Distillation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19316","snapshot_observed_at":"2026-08-07T14:01:04.861147Z","title":"Robust preference optimization through reward model distillation","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.861147Z"},"links":{"cited_paper":"/paper/2405.19316","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:4af91706054f8643762d4b96943c42f9954e6869fea214015666e9ca95628429","observation_id":"f499fe4f-e977-411a-88ef-e517db30b68f","resolution":{"observed_at":"2026-08-07T14:01:04.861147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.06810","last_updated":"2025-03-10T00:13:19Z","snapshot_observed_at":"2026-08-07T17:18:24.160798Z","submitted_at":"2025-03-10T00:13:19Z","title":"Mitigating Preference Hacking in Policy Optimization with Pessimism","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.06810","snapshot_observed_at":"2026-08-07T14:01:04.976843Z","title":"Mitigating preference hacking in policy optimization with pessimism","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.976843Z"},"links":{"cited_paper":"/paper/2503.06810","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:c0330652d9e2654f2f14a6daf648beaafb6f777cc158f192553b18328be24606","observation_id":"1a636902-5b51-4f76-86a8-8078c4290c71","resolution":{"observed_at":"2026-08-07T14:01:04.976843Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.13399","last_updated":"2025-02-18T17:16:55Z","snapshot_observed_at":"2026-08-07T03:47:17.449911Z","submitted_at":"2024-07-18T11:08:40Z","title":"Correcting the Mythos of KL-Regularization: Direct Alignment without Overoptimization via Chi-Squared Preference Optimization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.13399","snapshot_observed_at":"2026-08-07T14:01:05.069235Z","title":"Correcting the mythos of kl-regularization: Direct alignment without overoptimization via chi-squared preference optimization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.069235Z"},"links":{"cited_paper":"/paper/2407.13399","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:c5dac967d76fb42fe89af2b4ec49627c3f935f9f08c0ba85694dfb62a9e77370","observation_id":"6c726fd7-51c0-4dcb-858c-88becab87c76","resolution":{"observed_at":"2026-08-07T14:01:05.069235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.17031","last_updated":"2024-03-24T02:59:27Z","snapshot_observed_at":"2026-07-06T17:50:26.908430Z","submitted_at":"2024-03-24T02:59:27Z","title":"The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.17031","snapshot_observed_at":"2026-08-07T14:01:05.147815Z","title":"The n+ implementation details of rlhf with ppo: A case study on tl;dr summarization, 2024b","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.147815Z"},"links":{"cited_paper":"/paper/2403.17031","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:badd222243e5d24616244b2ac4dd2d26b89e655a8a3a80de77cdee9fed7f0a5a","observation_id":"362722c1-ba46-4730-a3e0-bd411823339a","resolution":{"observed_at":"2026-08-07T14:01:05.147815Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10038","last_updated":"2024-03-30T16:10:47Z","snapshot_observed_at":"2026-07-06T17:30:43.666819Z","submitted_at":"2024-02-15T16:00:58Z","title":"RS-DPO: A Hybrid Rejection Sampling and Direct Preference Optimization Method for Alignment of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10038","snapshot_observed_at":"2026-08-07T14:01:05.235610Z","title":"Rs-dpo: A hybrid rejection sampling and direct preference optimization method for alignment of large language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.235610Z"},"links":{"cited_paper":"/paper/2402.10038","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:948c7073d2a7caeee7fe7d199164b99bcbde6d8abbd5380e535e9fae795b41e0","observation_id":"846f9a8e-399f-4bad-bf6b-ea1336025286","resolution":{"observed_at":"2026-08-07T14:01:05.235610Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.01643","last_updated":"2020-11-01T23:50:25Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-04T17:00:15Z","title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.01643","snapshot_observed_at":"2026-08-07T14:01:05.329676Z","title":"Offline reinforcement learning: Tutorial, review, and perspectives on open problems","venue":null,"work_id":null,"year":2005},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.329676Z"},"links":{"cited_paper":"/paper/2005.01643","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:f6b0b62c81e2e1a138f345d18276d0093a4938afcf0bffa5ee245f9075f9cc43","observation_id":"cc13c926-2db6-4395-8878-9f430c3aedc4","resolution":{"observed_at":"2026-08-07T14:01:05.329676Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.06657","last_updated":"2024-01-23T23:16:11Z","snapshot_observed_at":"2026-07-31T03:53:01.450978Z","submitted_at":"2023-09-13T01:07:25Z","title":"Statistical Rejection Sampling Improves Preference Optimization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.06657","snapshot_observed_at":"2026-08-07T14:01:05.396920Z","title":"Statistical rejection sampling improves preference optimization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.396920Z"},"links":{"cited_paper":"/paper/2309.06657","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:1310bed9b1267f232abe9080a8b32bf83cbed355c1c5cb629fac9c40c148f1e8","observation_id":"10740686-15de-458a-8a08-82ef54fc1c73","resolution":{"observed_at":"2026-08-07T14:01:05.396920Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.13156","last_updated":"2025-02-27T16:30:42Z","snapshot_observed_at":"2026-07-06T19:18:27.644746Z","submitted_at":"2024-09-20T01:46:07Z","title":"RRM: Robust Reward Model Training Mitigates Reward Hacking","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.13156","snapshot_observed_at":"2026-08-07T14:01:05.490143Z","title":"Rrm: Robust reward model training mitigates reward hacking","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.490143Z"},"links":{"cited_paper":"/paper/2409.13156","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:21bea7ebbeabc7a493ad0f93da376bcf0e786dfd1a4d8068582d6453babe9df5","observation_id":"3e72c6b5-9edd-4f59-b902-d1674575148e","resolution":{"observed_at":"2026-08-07T14:01:05.490143Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.12187","last_updated":"2024-01-22T18:27:08Z","snapshot_observed_at":"2026-07-30T18:16:51.527587Z","submitted_at":"2024-01-22T18:27:08Z","title":"WARM: On the Benefits of Weight Averaged Reward Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.12187","snapshot_observed_at":"2026-08-07T14:01:05.649860Z","title":"Warm: On the benefits of weight averaged reward models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.649860Z"},"links":{"cited_paper":"/paper/2401.12187","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:a0c4b3b1c9476daff04d0d399ce35db451da4e1c86cfe19dd7fe01f0b1354677","observation_id":"a2b21913-dd8a-401e-bf9e-3fe44b655998","resolution":{"observed_at":"2026-08-07T14:01:05.649860Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.03715","last_updated":"2024-04-04T17:56:41Z","snapshot_observed_at":"2026-08-05T22:29:52.212015Z","submitted_at":"2024-04-04T17:56:41Z","title":"Direct Nash Optimization: Teaching Language Models to Self-Improve with General Preferences","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.03715","snapshot_observed_at":"2026-08-07T14:01:05.734257Z","title":"Direct nash optimization: Teaching language models to self-improve with general preferences","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.734257Z"},"links":{"cited_paper":"/paper/2404.03715","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:6b8ab81e704b4ea47feeb238b10f725a9e9ec0108a6ab7f3065e0ba3dfffa98c","observation_id":"cf583308-af18-4836-b38e-fa24ff026969","resolution":{"observed_at":"2026-08-07T14:01:05.734257Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-07T14:01:05.832817Z","title":"Proximal policy optimization algorithms","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.832817Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:a6806819b2d046e984d3d32b07503466be1d7f8bb9754bfcd4ae03163c30b4a6","observation_id":"dd3eca1a-614b-4892-a228-e5e9e4d1e819","resolution":{"observed_at":"2026-08-07T14:01:05.832817Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.16008","last_updated":"2025-03-14T20:08:08Z","snapshot_observed_at":"2026-07-06T18:50:22.540341Z","submitted_at":"2024-07-22T19:21:55Z","title":"Boosting Reward Model with Preference-Conditional Multi-Aspect Synthetic Data Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.16008","snapshot_observed_at":"2026-08-07T14:01:05.949991Z","title":"Boosting reward model with preference-conditional multi-aspect synthetic data generation","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.949991Z"},"links":{"cited_paper":"/paper/2407.16008","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:4aff439689e6b6f8091f7eb2b8a9eba02ccbce72f3e95c157cba896e32410edc","observation_id":"752f47ed-e83d-4753-81d5-9c3d5d35c94f","resolution":{"observed_at":"2026-08-07T14:01:05.949991Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.05749","last_updated":"2024-05-28T23:25:15Z","snapshot_observed_at":"2026-07-06T17:27:29.050237Z","submitted_at":"2024-02-08T15:33:09Z","title":"Generalized Preference Optimization: A Unified Approach to Offline Alignment","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.05749","snapshot_observed_at":"2026-08-07T14:01:06.110662Z","title":"Generalized preference optimization: A unified approach to offline alignment.arXiv preprint arXiv:2402.05749,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.110662Z"},"links":{"cited_paper":"/paper/2402.05749","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:36fd94ed578dbbc7e302bc6cfa1cd79fd07eea50424c7ce8282f131448a26b21","observation_id":"400d8bdf-8572-4ceb-ba4e-f027564c672b","resolution":{"observed_at":"2026-08-07T14:01:06.110662Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.06601","last_updated":"2023-03-18T20:44:45Z","snapshot_observed_at":"2026-08-03T12:43:10.574809Z","submitted_at":"2022-04-13T18:41:41Z","title":"Causal Confusion and Reward Misidentification in Preference-Based Reward Learning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.06601","snapshot_observed_at":"2026-08-07T14:01:06.259098Z","title":"Causal confusion and reward misidentification in preference-based reward learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.259098Z"},"links":{"cited_paper":"/paper/2204.06601","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:aa03424fa4f29bfa1c5aa9c922e21bee06cd9fca35e7068a504b920263a3cc19","observation_id":"2188078f-3ec4-47cd-8dcd-da45e6a778b8","resolution":{"observed_at":"2026-08-07T14:01:06.259098Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-07T14:01:06.403374Z","title":"Llama: Open and efficient foundation language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.403374Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:6d01746702e28fe8aad4dffc479a6cf189889942259e456bec895b68bec27841","observation_id":"efd341ea-140e-4c02-b775-92ebc86fb584","resolution":{"observed_at":"2026-08-07T14:01:06.403374Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.14554","last_updated":"2024-03-12T21:49:59Z","snapshot_observed_at":"2026-07-06T16:36:55.519052Z","submitted_at":"2023-10-23T04:19:35Z","title":"Making RL with Preference-based Feedback Efficient via Randomization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.14554","snapshot_observed_at":"2026-08-07T14:01:06.516230Z","title":"TL;DR: Mining Reddit to learn automatic summarization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.516230Z"},"links":{"cited_paper":"/paper/2310.14554","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:a75ad18b2ec40aa605de1f3f314cf1374218f08a324b267f939146326b3139d3","observation_id":"7defe75f-a49b-407f-84a5-9bff0f301c07","resolution":{"observed_at":"2026-08-07T14:01:06.516230Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.21046","last_updated":"2024-05-31T17:39:06Z","snapshot_observed_at":"2026-07-31T03:29:04.443362Z","submitted_at":"2024-05-31T17:39:06Z","title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.21046","snapshot_observed_at":"2026-08-07T14:01:06.669955Z","title":"Exploratory preference optimization: Harnessing implicit q*-approximation for sample-efficient rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.669955Z"},"links":{"cited_paper":"/paper/2405.21046","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:ee8efff1520a3fff543259f1f93cfe2c12dd20f0e42bd0fedbf9819ad9fc3391","observation_id":"f7e99903-dfd7-4e45-a6ac-6fd3782b22ad","resolution":{"observed_at":"2026-08-07T14:01:06.669955Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11456","last_updated":"2024-05-01T14:50:56Z","snapshot_observed_at":"2026-08-02T03:59:22.576409Z","submitted_at":"2023-12-18T18:58:42Z","title":"Iterative Preference Learning from Human Feedback: Bridging Theory and Practice for RLHF under KL-Constraint","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11456","snapshot_observed_at":"2026-08-07T14:01:06.783481Z","title":"Iterative preference learning from human feedback: Bridging theory and practice for rlhf under kl-constraint","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.783481Z"},"links":{"cited_paper":"/paper/2312.11456","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:b73ef0cd961ab043d1ce1943bc7c3107ec860602d348597137a6d2fd6c380f85","observation_id":"c925f773-402e-4005-aad0-6c67f4ba00c6","resolution":{"observed_at":"2026-08-07T14:01:06.783481Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14816","last_updated":"2023-09-29T19:18:55Z","snapshot_observed_at":"2026-07-06T15:32:18.583350Z","submitted_at":"2023-05-24T07:11:26Z","title":"Provable Offline Preference-Based Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14816","snapshot_observed_at":"2026-08-07T14:01:06.936809Z","title":"Provable offline preference-based reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:06.936809Z"},"links":{"cited_paper":"/paper/2305.14816","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:da7a6e33abb627f9fb50d16c5dbfcbea30d73a098b538a87593a3eb907498430","observation_id":"a3074639-539a-49a7-97b8-496c5ef4f467","resolution":{"observed_at":"2026-08-07T14:01:06.936809Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19332","last_updated":"2024-11-05T07:21:18Z","snapshot_observed_at":"2026-07-06T18:22:10.094519Z","submitted_at":"2024-05-29T17:59:07Z","title":"Self-Exploring Language Models: Active Preference Elicitation for Online Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19332","snapshot_observed_at":"2026-08-07T14:01:07.063483Z","title":"Self-exploring language models: Active preference elicitation for online alignment","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:07.063483Z"},"links":{"cited_paper":"/paper/2405.19332","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:77731a695c7b7ed7a6c811268f4a4bd83e838737a8e504984982179162fe240c","observation_id":"9a3474c4-bb56-4ffb-89d4-bf52cc09386e","resolution":{"observed_at":"2026-08-07T14:01:07.063483Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:01:07.182899Z","title":"Logarithmic regret for online kl-regularized reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:07.182899Z"},"links":{"citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:0744071b144c9aad8b1350ccb3fa3454e42e2ee09a730e5c5cbf181a9515cb3b","observation_id":"43c8e79b-099b-42df-b0df-f7e4b8b3a09b","resolution":{"observed_at":"2026-08-07T14:01:07.182899Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:01:08.799954Z","title":"1 will increase the bound on the performance gap analysis, which is undesired for sampling efficiency","venue":null,"work_id":"f32b2fc5-7498-4888-96a1-599c0c1748b2","year":2023},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:07.361030Z"},"links":{"citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:06cd6ef8963f42ef2a7357cd2caffff0cac30c6c2def1de4e1f41252b75c5ada","observation_id":"192b52ba-3723-4214-ab5e-f95fd3bc5dcf","resolution":{"observed_at":"2026-08-07T14:01:08.907969Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19320","last_updated":"2025-02-19T00:51:58Z","snapshot_observed_at":"2026-07-06T18:22:10.094519Z","submitted_at":"2024-05-29T17:51:42Z","title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19320","snapshot_observed_at":"2026-08-07T14:01:04.563716Z","title":"Value-incentivized preference optimization: A unified approach to online and offline rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.563716Z"},"links":{"cited_paper":"/paper/2405.19320","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:4cfbcdc03e921a70766ec11192b1059c5664877acbf583c76f3e9f6c2a1a3c4c","observation_id":"df8a4b7b-f1ed-41fc-98b0-148f8d11c3fb","resolution":{"observed_at":"2026-08-07T14:01:04.563716Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.18130","last_updated":"2025-03-23T16:20:59Z","snapshot_observed_at":"2026-08-07T16:43:04.031245Z","submitted_at":"2025-03-23T16:20:59Z","title":"Mitigating Reward Over-Optimization in RLHF via Behavior-Supported Regularization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.18130","snapshot_observed_at":"2026-08-07T14:01:04.624213Z","title":"Mitigating reward over- optimization in rlhf via behavior-supported regularization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.624213Z"},"links":{"cited_paper":"/paper/2503.18130","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:6a424b9556884b65be3fcb809633118c9fa4eb8ad8bbfbe1617326d63ce35d9c","observation_id":"3d243c16-4172-4782-96f3-490df15f7716","resolution":{"observed_at":"2026-08-07T14:01:04.624213Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.05434","last_updated":"2025-08-08T03:36:58Z","snapshot_observed_at":"2026-07-06T20:33:11.256513Z","submitted_at":"2025-02-08T03:47:00Z","title":"Sample-Efficient Reinforcement Learning from Human Feedback via Information-Directed Sampling","version":3},"cited_work":{"arxiv_id":"2502.05434","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.05434","snapshot_observed_at":"2026-08-07T14:01:08.101786Z","title":"Sample-Efficient Reinforcement Learning from Human Feedback via Information-Directed Sampling","venue":"cs.LG","work_id":"bd3c668e-b165-43e9-b401-bb3fe5f0d4de","year":2025},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:05.568274Z"},"links":{"cited_paper":"/paper/2502.05434","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:2e4f9ad593d355e83583d2745469fed801389d27561b226e55aeab60beb76bce","observation_id":"301e7a58-e6e9-434e-9935-0ccf2266b1bf","resolution":{"observed_at":"2026-08-07T14:01:08.191682Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2304.01373","last_updated":"2023-05-31T17:54:07Z","snapshot_observed_at":"2026-08-07T23:49:29.478548Z","submitted_at":"2023-04-03T20:58:15Z","title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.01373","snapshot_observed_at":"2026-08-07T14:01:04.472540Z","title":"Ralph Allan Bradley and Milton E Terry","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.472540Z"},"links":{"cited_paper":"/paper/2304.01373","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:43d6e0cefde1a8bb9be2e910e543b44e5010e77dfe491ee077ed4011e2b601ef","observation_id":"44fed6cc-8e4a-4b34-b7fe-75eaa37a3fbd","resolution":{"observed_at":"2026-08-07T14:01:04.472540Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-07T14:01:04.407989Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.407989Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:9718f71990d3bc41b80468105936a84d2cffc91a2602e05edd9dbaf0f3de272e","observation_id":"8a6d0d9c-1030-42b3-add9-adf75ffc89b3","resolution":{"observed_at":"2026-08-07T14:01:04.407989Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.07863","last_updated":"2024-11-12T11:18:43Z","snapshot_observed_at":"2026-08-07T08:32:00.916309Z","submitted_at":"2024-05-13T15:50:39Z","title":"RLHF Workflow: From Reward Modeling to Online RLHF","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.07863","snapshot_observed_at":"2026-08-07T14:01:04.698570Z","title":"Rlhf workflow: From reward modeling to online rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF","version":1},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:04.698570Z"},"links":{"cited_paper":"/paper/2405.07863","citing_paper":"/paper/2505.20556"},"observation_digest":"sha256:38f54c0431dfd69063fba2fd21090977863634ec083ccb1f15329d28854075c2","observation_id":"81bbaf25-9f91-4f8c-a513-b6862e9d21ba","resolution":{"observed_at":"2026-08-07T14:01:04.698570Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.20556","last_updated":"2025-05-26T22:34:42Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-07T13:49:33.928724Z","submitted_at":"2025-05-26T22:34:42Z","title":"Learning a Pessimistic Reward Model in RLHF"},"reference_resolution":{"displayed":31,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":27,"verified_exact":1,"verified_fuzzy":3},"total_outbound_references":31},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 31 of 31 outbound references and 3 inbound Pith citation observations for arXiv:2505.20556."}