{"as_of":"2026-08-09T22:40:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:bb70c0d44edc990ccde925fbee5da157f3a5561ebcc986ecae8555534b4bcac9","coverage":[{"denominator":21,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":21,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T19:40:42.080296Z","state":"measured"},{"denominator":23,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":23,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-21T13:06:54.002248Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-21T13:10:10.404876Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"cited_work":{"arxiv_id":"2502.06861","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2502.06861","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Design considerations in offline preference-based rl","venue":null,"work_id":"09c3c6cb-11d2-44d6-a1ad-296f9a473b37","year":null},"citing_paper":{"arxiv_id":"2602.06239","last_updated":"2026-05-15T16:18:46Z","snapshot_observed_at":"2026-07-06T22:44:46.690576Z","submitted_at":"2026-02-05T22:31:07Z","title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-05-16T06:35:30.479542Z"},"links":{"cited_paper":"/paper/2502.06861","citing_paper":"/paper/2602.06239"},"observation_digest":"sha256:bafddf6769b082122114812e4a59e1a66552324a0638388c1294447ba355017d","observation_id":"3fe1c548-1e61-48a0-9c5f-c0a9d67d8b9e","resolution":{"observed_at":"2026-05-16T06:37:28.516891Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"cited_work":{"arxiv_id":"2502.06861","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2502.06861","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Design considerations in offline preference-based rl","venue":null,"work_id":"09c3c6cb-11d2-44d6-a1ad-296f9a473b37","year":null},"citing_paper":{"arxiv_id":"2602.06239","last_updated":"2026-05-15T16:18:46Z","snapshot_observed_at":"2026-07-06T22:44:46.690576Z","submitted_at":"2026-02-05T22:31:07Z","title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","version":3},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-05-21T13:06:54.002248Z"},"links":{"cited_paper":"/paper/2502.06861","citing_paper":"/paper/2602.06239"},"observation_digest":"sha256:2b7353863e8b9960706722837ab77f1aae8a2e8cd7bb6021335e72977fe5758f","observation_id":"6a5f2044-b3f1-4b49-9a79-dc6d9ad3e799","resolution":{"observed_at":"2026-05-21T13:10:10.406878Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2502.06861/citation-record","integrity":"/paper/2502.06861/integrity","json":"/paper/2502.06861/citation-record.json","paper":"/paper/2502.06861"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2402.14740","last_updated":"2024-02-26T18:26:25Z","snapshot_observed_at":"2026-08-09T14:30:33.899591Z","submitted_at":"2024-02-22T17:52:34Z","title":"Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14740","snapshot_observed_at":"2026-08-08T19:40:41.977517Z","title":"Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:41.977517Z"},"links":{"cited_paper":"/paper/2402.14740","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:be1eafa9c80f4b6212e03bf0f9ec12b2eb771a26867da399e0f05449a0593119","observation_id":"18182556-cba3-4140-bd73-4b789af87ceb","resolution":{"observed_at":"2026-08-08T19:40:41.977517Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T19:40:42.410863Z","title":null,"venue":null,"work_id":"32892822-0bdf-45d1-b147-43e6d44d98ab","year":2017},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.074455Z"},"links":{"citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:23e72fdd630c64f41e75b6dc2947b04ca60fbba71a3c82c9f940932f5af5d84f","observation_id":"5c30b055-fb90-4bf3-8124-fbb6fabce50c","resolution":{"observed_at":"2026-08-08T19:40:42.415603Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19316","last_updated":"2025-03-03T08:22:25Z","snapshot_observed_at":"2026-08-02T12:00:09.704648Z","submitted_at":"2024-05-29T17:39:48Z","title":"Robust Preference Optimization through Reward Model Distillation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19316","snapshot_observed_at":"2026-08-08T19:40:41.999377Z","title":"Robust preference optimization through reward model distillation","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:41.999377Z"},"links":{"cited_paper":"/paper/2405.19316","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:4c09415948321fc6d00340eaa8f3fa43076b2b1e7b5c2269aabdb92ae3188748","observation_id":"89e4aabb-baae-45b5-af0f-9e2d9e9d6917","resolution":{"observed_at":"2026-08-08T19:40:41.999377Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.13399","last_updated":"2025-02-18T17:16:55Z","snapshot_observed_at":"2026-08-07T03:47:17.449911Z","submitted_at":"2024-07-18T11:08:40Z","title":"Correcting the Mythos of KL-Regularization: Direct Alignment without Overoptimization via Chi-Squared Preference Optimization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.13399","snapshot_observed_at":"2026-08-08T19:40:42.004209Z","title":"D., Sun, W., Krish- namurthy, A., and Foster, D","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.004209Z"},"links":{"cited_paper":"/paper/2407.13399","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:a862cad279605ee40b55f348db7b7ab7919ea93ea13da7be4eb1c0cabac1d432","observation_id":"28b3bc5c-75f7-4b8e-bf00-8d71f0b107ff","resolution":{"observed_at":"2026-08-08T19:40:42.004209Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.16436","last_updated":"2024-12-04T08:15:35Z","snapshot_observed_at":"2026-07-06T18:20:01.382394Z","submitted_at":"2024-05-26T05:38:50Z","title":"Provably Mitigating Overoptimization in RLHF: Your SFT Loss is Implicitly an Adversarial Regularizer","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.16436","snapshot_observed_at":"2026-08-08T19:40:42.009598Z","title":"Provably mitigating overopti- mization in rlhf: Your sft loss is implicitly an adversarial regularizer","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.009598Z"},"links":{"cited_paper":"/paper/2405.16436","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:76cf2d1422aac0fce1c79a165ad1a66165973f5dba2faaf03c255cc6f1219086","observation_id":"8e476658-38ac-4c5e-8c3c-b49441490b4c","resolution":{"observed_at":"2026-08-08T19:40:42.009598Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.00886","last_updated":"2024-06-11T16:25:52Z","snapshot_observed_at":"2026-07-06T16:55:53.666800Z","submitted_at":"2023-12-01T19:26:23Z","title":"Nash Learning from Human Feedback","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.00886","snapshot_observed_at":"2026-08-08T19:40:42.019294Z","title":"G., Row- land, M., Guo, Z","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.019294Z"},"links":{"cited_paper":"/paper/2312.00886","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:dd3b07f6f5ccb367d2996e723c6ede9576169bbb35eb61e3aa3022754ac00206","observation_id":"9ba1fac7-9632-4932-a5e4-4a15ee9fb486","resolution":{"observed_at":"2026-08-08T19:40:42.019294Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.19159","last_updated":"2024-09-09T04:39:31Z","snapshot_observed_at":"2026-08-08T10:19:27.268855Z","submitted_at":"2024-03-28T06:03:47Z","title":"Disentangling Length from Quality in Direct Preference Optimization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.19159","snapshot_observed_at":"2026-08-08T19:40:42.029675Z","title":"Disentan- gling length from quality in direct preference optimiza- tion","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.029675Z"},"links":{"cited_paper":"/paper/2403.19159","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:80e7746cc583fc404e759f22bce6f0deb20e51ee958930a5bf34fbd3641c14d7","observation_id":"c94c4030-9263-4763-91fc-8508caf05beb","resolution":{"observed_at":"2026-08-08T19:40:42.029675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.02900","last_updated":"2024-11-05T01:44:14Z","snapshot_observed_at":"2026-07-06T18:25:35.827334Z","submitted_at":"2024-06-05T03:41:37Z","title":"Scaling Laws for Reward Model Overoptimization in Direct Alignment Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.02900","snapshot_observed_at":"2026-08-08T19:40:42.034684Z","title":"Scaling laws for reward model overoptimization in direct alignment algo- rithms","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.034684Z"},"links":{"cited_paper":"/paper/2406.02900","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:1f2b2108aca17e29759356b4b7c028c3fc4879f1899a7f46efeaacc88a304d8d","observation_id":"5f84b809-9c0b-464b-9ae6-7e16dff56c16","resolution":{"observed_at":"2026-08-08T19:40:42.034684Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-08T19:40:42.049982Z","title":"M., Hauth, A., et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.049982Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:1f0250eadfbe3f708fa3378e043d9218d2a530d299084efc3c11402aeed6ce05","observation_id":"4639f06b-92d8-410e-aac7-ed4c6e12b946","resolution":{"observed_at":"2026-08-08T19:40:42.049982Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.14111","last_updated":"2023-11-03T18:36:03Z","snapshot_observed_at":"2026-07-06T15:46:21.260261Z","submitted_at":"2023-06-25T03:18:15Z","title":"Is RLHF More Difficult than Standard RL?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.14111","snapshot_observed_at":"2026-08-08T19:40:42.054396Z","title":"Is rlhf more difficult than standard rl? arXiv preprint arXiv:2306.14111,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.054396Z"},"links":{"cited_paper":"/paper/2306.14111","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:de5bf3f6fe515f352511a135604143cdda155f48120b1571f7e9b44d6b2a183e","observation_id":"a64d54ee-d14d-4af9-a98e-368242dbd36e","resolution":{"observed_at":"2026-08-08T19:40:42.054396Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.10425","last_updated":"2023-05-17T17:57:10Z","snapshot_observed_at":"2026-08-02T06:11:08.013358Z","submitted_at":"2023-05-17T17:57:10Z","title":"SLiC-HF: Sequence Likelihood Calibration with Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.10425","snapshot_observed_at":"2026-08-08T19:40:42.069597Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.069597Z"},"links":{"cited_paper":"/paper/2305.10425","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:3bdef3d0a8d6f13aad6c7e28a9160ba924a80e58d1ced1bb06641a6eb8c586f1","observation_id":"3c00348f-ff9c-439e-a0ce-a04c3d03c991","resolution":{"observed_at":"2026-08-08T19:40:42.069597Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T19:40:42.392687Z","title":"The optimizer used is Adafactor with learning rate that is constant with a linear warm-up for 2000 steps and a base rate of 1e −","venue":null,"work_id":"ebbf1097-5c6c-404d-848f-1a3254df9200","year":2000},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.080296Z"},"links":{"citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:ae67bc18e7bb91f919d6fc0346e17ffc5e5f782b12bcd2e0c299495f71b66840","observation_id":"5641bb44-ced0-44c8-8857-b9c0b6d82cc5","resolution":{"observed_at":"2026-08-08T19:40:42.399596Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19320","last_updated":"2025-02-19T00:51:58Z","snapshot_observed_at":"2026-07-06T18:22:10.094519Z","submitted_at":"2024-05-29T17:51:42Z","title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19320","snapshot_observed_at":"2026-08-08T19:40:41.989242Z","title":"Value-incentivized preference optimization: A unified approach to online and offline rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:41.989242Z"},"links":{"cited_paper":"/paper/2405.19320","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:bd521b15b127fef14b76f8cf2b6cb02c95945c574210a71dacfc187033a587fc","observation_id":"02e5e5a0-dffe-4b30-822d-13b599384c68","resolution":{"observed_at":"2026-08-08T19:40:41.989242Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.00045","last_updated":"2022-09-30T19:16:16Z","snapshot_observed_at":"2026-07-31T07:22:44.364265Z","submitted_at":"2022-09-30T19:16:16Z","title":"Calibrating Sequence likelihood Improves Conditional Language Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.00045","snapshot_observed_at":"2026-08-08T19:40:42.064438Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2007,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.064438Z"},"links":{"cited_paper":"/paper/2210.00045","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:9a8e905db5c85577dab6dec1578c5a96431f68ae4af6978ceb1199020a8f27a9","observation_id":"e8739783-b312-4542-905a-f34c8373c5f4","resolution":{"observed_at":"2026-08-08T19:40:42.064438Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04425","last_updated":"2026-07-03T03:04:48Z","snapshot_observed_at":"2026-08-04T06:53:56.097854Z","submitted_at":"2024-06-06T18:10:51Z","title":"On Regularization via Early Stopping for Least Squares Regression","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04425","snapshot_observed_at":"2026-08-08T19:40:42.039660Z","title":"On regularization via early stopping for least squares regression","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2010,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.039660Z"},"links":{"cited_paper":"/paper/2406.04425","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:4efb846af3fa33ec8ef02b8f7b96db99ea7dad33f3d99f8b92deb8d5c0d98dbf","observation_id":"a37fcd4e-f981-4ab9-8bd2-ce2ed2866e01","resolution":{"observed_at":"2026-08-08T19:40:42.039660Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14734","last_updated":"2024-11-01T20:05:19Z","snapshot_observed_at":"2026-08-03T02:02:15.325394Z","submitted_at":"2024-05-23T16:01:46Z","title":"SimPO: Simple Preference Optimization with a Reference-Free Reward","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14734","snapshot_observed_at":"2026-08-08T19:40:42.014455Z","title":"URL https://books","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2012,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.014455Z"},"links":{"cited_paper":"/paper/2405.14734","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:8af82d53d4d5594db32f754ddcc242e2c06a9f328d3814a50f186e6cba50ef41","observation_id":"caabea31-3e40-41bf-b61e-2ab71ce38f96","resolution":{"observed_at":"2026-08-08T19:40:42.014455Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01306","last_updated":"2024-11-19T18:12:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-02-02T10:53:36Z","title":"KTO: Model Alignment as Prospect Theoretic Optimization","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01306","snapshot_observed_at":"2026-08-08T19:40:41.994465Z","title":"KTO: Model alignment as prospect theoretic optimization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:41.994465Z"},"links":{"cited_paper":"/paper/2402.01306","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:9604c5d5af0b9a575e6c503e72c269aa3bcbc1a0457d7736885591fcb54292ba","observation_id":"d202202d-2a25-428d-938b-5161964e17b7","resolution":{"observed_at":"2026-08-08T19:40:41.994465Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04056","last_updated":"2024-06-13T14:45:36Z","snapshot_observed_at":"2026-08-09T09:34:04.204036Z","submitted_at":"2024-01-08T17:55:02Z","title":"A Minimaximalist Approach to Reinforcement Learning from Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04056","snapshot_observed_at":"2026-08-08T19:40:42.044924Z","title":"S., and Agarwal, A","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.044924Z"},"links":{"cited_paper":"/paper/2401.04056","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:215275431250d2cb83239a677f7b4b5d0d184dab7fbd0cdf310ddcb7955c06ee","observation_id":"1daba942-17dd-4a26-bfe4-8731dcd449fd","resolution":{"observed_at":"2026-08-08T19:40:42.044924Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.08417","last_updated":"2024-06-03T01:28:06Z","snapshot_observed_at":"2026-07-06T17:16:13.055978Z","submitted_at":"2024-01-16T15:04:51Z","title":"Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.08417","snapshot_observed_at":"2026-08-08T19:40:42.059588Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.059588Z"},"links":{"cited_paper":"/paper/2401.08417","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:05dc79ebb59cc4b8469bef2f98544de87616421033d42f0f1bbaf572c132fd85","observation_id":"7d1f0652-e606-4e17-8087-744fdbbf5409","resolution":{"observed_at":"2026-08-08T19:40:42.059588Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.13228","last_updated":"2024-07-03T13:46:33Z","snapshot_observed_at":"2026-08-08T16:03:10.053914Z","submitted_at":"2024-02-20T18:42:34Z","title":"Smaug: Fixing Failure Modes of Preference Optimisation with DPO-Positive","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.13228","snapshot_observed_at":"2026-08-08T19:40:42.024176Z","title":"Smaug: Fixing failure modes of pref- erence optimisation with dpo-positive","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:42.024176Z"},"links":{"cited_paper":"/paper/2402.13228","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:f0178bb409884884bac945c09e6f5e4238addab1cf3800efc1c72d0d3bed4da5","observation_id":"89d56662-a529-4de4-8ddc-6401e9a9a7d6","resolution":{"observed_at":"2026-08-08T19:40:42.024176Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10571","last_updated":"2024-06-06T12:02:37Z","snapshot_observed_at":"2026-07-06T17:31:04.675896Z","submitted_at":"2024-02-16T10:55:38Z","title":"Direct Preference Optimization with an Offset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10571","snapshot_observed_at":"2026-08-08T19:40:41.983451Z","title":"Direct pref- erence optimization with an offset","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-08T19:40:41.983451Z"},"links":{"cited_paper":"/paper/2402.10571","citing_paper":"/paper/2502.06861"},"observation_digest":"sha256:d96c36be3b77680e32d8eaf6f6d453bca02f2542666bc16f2b12dab02b1cdfa7","observation_id":"09c3038c-a61c-4307-9b25-f209c322ef1c","resolution":{"observed_at":"2026-08-08T19:40:41.983451Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2502.06861","last_updated":"2025-02-08T00:01:37Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-08T19:30:36.087038Z","submitted_at":"2025-02-08T00:01:37Z","title":"Design Considerations in Offline Preference-based RL"},"reference_resolution":{"displayed":21,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":20,"verified_exact":0,"verified_fuzzy":1},"total_outbound_references":21},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 21 of 21 outbound references and 2 inbound Pith citation observations for arXiv:2502.06861."}