{"as_of":"2026-08-17T20:14:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:9051ef676775720fcb3ea5799981a91881293e14f4481148f116221f5cfaed2c","coverage":[{"denominator":18,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":18,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T04:53:39.197075Z","state":"measured"},{"denominator":32,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":32,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":14,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":14,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-14T04:16:17.105396Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-04T20:40:07.862245Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-08-04T22:59:14.560552Z","title":"On a few pitfalls in kl divergence gradient estimation for rl.arXiv preprint arXiv:2506.09477,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.06941","last_updated":"2025-09-08T17:52:56Z","snapshot_observed_at":"2026-08-07T12:11:53.268632Z","submitted_at":"2025-09-08T17:52:56Z","title":"Outcome-based Exploration for LLM Reasoning","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-04T22:59:14.560552Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2509.06941"},"observation_digest":"sha256:061b8b47d64eac2fea88940aa8471d485b0a793c9eb37c721518a1a8d7a89f7d","observation_id":"fb93e9e2-8fda-416f-af68-8a2d11cd188d","resolution":{"observed_at":"2026-08-04T22:59:14.560552Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2601.16175","last_updated":"2026-02-05T18:03:03Z","snapshot_observed_at":"2026-08-01T10:41:29.420510Z","submitted_at":"2026-01-22T18:24:00Z","title":"Learning to Discover at Test Time","version":2},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-05-16T05:16:04.001700Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2601.16175"},"observation_digest":"sha256:dfcc2b239af82582693c1f51d2ff732374ff9884fcf170354d112784dd6017dd","observation_id":"6d111ca8-cc59-4718-882c-f33a1c380929","resolution":{"observed_at":"2026-05-16T05:16:04.236306Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2604.10674","last_updated":"2026-04-12T14:57:52Z","snapshot_observed_at":"2026-08-14T14:15:55.861698Z","submitted_at":"2026-04-12T14:57:52Z","title":"Skill-SD: Skill-Conditioned Self-Distillation for Multi-turn LLM Agents","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-05-10T15:28:07.981488Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2604.10674"},"observation_digest":"sha256:71191fe010753d9c2e5f9767f36b046aa468677113ccfa23f8557583f8f92e3f","observation_id":"2ddf49ea-7940-4f64-adfa-cd484f4b793b","resolution":{"observed_at":"2026-05-11T10:31:00.921568Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2604.16259","last_updated":"2026-04-17T17:17:55Z","snapshot_observed_at":"2026-08-12T19:06:49.955269Z","submitted_at":"2026-04-17T17:17:55Z","title":"Beyond Distribution Sharpening: The Importance of Task Rewards","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-05-10T08:07:14.691463Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2604.16259"},"observation_digest":"sha256:4a8cd1b2e92ae890a0f1c1e9b7312b16fa9130f4dd899f18302aa80f3fd29d53","observation_id":"f7149b28-9cc6-495b-b573-031bda18baad","resolution":{"observed_at":"2026-05-10T09:08:26.980834Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2605.07865","last_updated":"2026-05-08T15:24:51Z","snapshot_observed_at":"2026-08-15T16:24:55.947978Z","submitted_at":"2026-05-08T15:24:51Z","title":"KL for a KL: On-Policy Distillation with Control Variate Baseline","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-05-11T03:31:07.462474Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2605.07865"},"observation_digest":"sha256:0374a7b22c81a0b514f26eb8186e8db275e5ba1dfe3a9d610d53df28e957f930","observation_id":"8be8f200-8671-4822-9aa1-4abcfff1152f","resolution":{"observed_at":"2026-05-11T03:40:54.210286Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2605.09214","last_updated":"2026-05-09T23:17:46Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2026-05-09T23:17:46Z","title":"Fast Rates for Offline Contextual Bandits with Forward-KL Regularization under Single-Policy Concentrability","version":1},"reference_index":111,"source":"arxiv_source","source_observed_at":"2026-05-12T03:47:14.379908Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2605.09214"},"observation_digest":"sha256:49efb047e31b7fdb65529bbb0508631e39deb408286691e3125a1e471b960254","observation_id":"21fb574d-adc8-4751-984e-03958d073072","resolution":{"observed_at":"2026-05-12T06:56:30.653037Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2605.21605","last_updated":"2026-05-22T02:16:59Z","snapshot_observed_at":"2026-08-16T04:29:19.461099Z","submitted_at":"2026-05-20T18:12:29Z","title":"GenEvolve: Self-Evolving Image Generation Agents via Tool-Orchestrated Visual Experience Distillation","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-05-22T09:29:05.360018Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2605.21605"},"observation_digest":"sha256:80c76549e40e416113c6fa61b1bde7a2ea3b59735f6c4a273986f2005e441757","observation_id":"6dedc3fd-0a6b-47c4-acca-3d86a6e5e0b1","resolution":{"observed_at":"2026-05-22T09:31:23.051850Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2605.21605","last_updated":"2026-05-22T02:16:59Z","snapshot_observed_at":"2026-08-16T04:29:19.461099Z","submitted_at":"2026-05-20T18:12:29Z","title":"GenEvolve: Self-Evolving Image Generation Agents via Tool-Orchestrated Visual Experience Distillation","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-05-25T05:38:33.676797Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2605.21605"},"observation_digest":"sha256:4816956a1e4b405d5d1ed757b7fbe25d825077361d4f7c829a936e6d50f8ad69","observation_id":"ec98e39a-e769-499a-81cf-4147de42a357","resolution":{"observed_at":"2026-05-25T05:40:24.070676Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2606.01039","last_updated":"2026-05-31T06:10:38Z","snapshot_observed_at":"2026-08-13T15:03:20.418417Z","submitted_at":"2026-05-31T06:10:38Z","title":"OPD+: Rethinking the Advantage Design for On-Policy Distillation","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-06-28T17:24:21.906134Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2606.01039"},"observation_digest":"sha256:f0cc38e96dcec3e409b397c85e97f7e369c335f3db24ee78cc51bb98eea65db6","observation_id":"2041392a-b305-4125-865c-1ab986934b85","resolution":{"observed_at":"2026-07-01T21:16:13.569504Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2606.03382","last_updated":"2026-06-04T22:09:00Z","snapshot_observed_at":"2026-08-02T02:33:38.414458Z","submitted_at":"2026-06-02T09:26:26Z","title":"Local Guidance, Global Impact: Gaussian-Reshaped Trust Region Unlocks Behavior Transitions","version":2},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-06-28T10:51:30.546134Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2606.03382"},"observation_digest":"sha256:43ee267a1fc8fa23ea65d07f68dd73d5140a13b6cd0e9d978dc8f61f24a0c70f","observation_id":"8a1953ca-efd8-4d3e-a81f-03c8585f6e50","resolution":{"observed_at":"2026-07-02T02:36:26.596091Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":"2506.09477","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-04T20:40:07.862245Z","title":"On a few pitfalls in kl divergence gradient estimation for rl","venue":null,"work_id":"ff22cdf8-f3a8-467c-8e9d-2136db7a660b","year":2025},"citing_paper":{"arxiv_id":"2606.26080","last_updated":"2026-06-24T17:54:08Z","snapshot_observed_at":"2026-08-02T15:38:18.729836Z","submitted_at":"2026-06-24T17:54:08Z","title":"Neglected Free Lunch from Post-training: Progress Advantage for LLM Agents","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-06-25T19:55:51.114244Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2606.26080"},"observation_digest":"sha256:8a1ffe0fbc891ce60369c48c9b7831dc346ea1071ab60acae422105db17c3d50","observation_id":"cf6f60d5-6cde-45e3-8460-3ab7a060b84b","resolution":{"observed_at":"2026-07-04T20:40:07.863689Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-07-14T11:23:30.063231Z","title":"arXiv preprint arXiv:2506.09477 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10481","last_updated":"2026-07-21T15:48:36Z","snapshot_observed_at":"2026-08-14T18:58:02.664803Z","submitted_at":"2026-07-11T21:22:46Z","title":"ARMOR: Stabilizing On-Policy LLM RL with Off-Policy Anchor Samples","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-07-14T11:23:30.063231Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2607.10481"},"observation_digest":"sha256:d2902b8ff246b1a29423e0300ce19ce355d5db07b86bdd11c858f4a78d9826cf","observation_id":"e19c15c6-40ff-48eb-a0e2-3d112c0596d3","resolution":{"observed_at":"2026-07-14T11:23:30.063231Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-08-02T07:23:27.338911Z","title":"arXiv preprint arXiv:2506.09477 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10481","last_updated":"2026-07-21T15:48:36Z","snapshot_observed_at":"2026-08-14T18:58:02.664803Z","submitted_at":"2026-07-11T21:22:46Z","title":"ARMOR: Stabilizing On-Policy LLM RL with Off-Policy Anchor Samples","version":2},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-02T07:23:27.338911Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2607.10481"},"observation_digest":"sha256:19bbb1df276f02858b285516e4034cf9c686048f392add0feb8837a26c05939e","observation_id":"52d4cd12-08db-4a46-9185-101a8416eb73","resolution":{"observed_at":"2026-08-02T07:23:27.338911Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.09477","snapshot_observed_at":"2026-08-14T04:16:17.105396Z","title":"On a few pitfalls in","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.10260","last_updated":"2026-08-10T21:49:45Z","snapshot_observed_at":"2026-08-17T06:56:32.154760Z","submitted_at":"2026-08-10T21:49:45Z","title":"Interpreting Language Model Hidden States at Scale","version":1},"reference_index":106,"source":"arxiv_source","source_observed_at":"2026-08-14T04:16:17.105396Z"},"links":{"cited_paper":"/paper/2506.09477","citing_paper":"/paper/2608.10260"},"observation_digest":"sha256:03c40adab421c1284a16c3a92fc87a0dc782b358d013d43ac23ada503b2465ae","observation_id":"f0d6b363-5e05-47cc-8ecf-0ae3bcc016e7","resolution":{"observed_at":"2026-08-14T04:16:17.105396Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2506.09477/citation-record","integrity":"/paper/2506.09477/integrity","json":"/paper/2506.09477/citation-record.json","paper":"/paper/2506.09477"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2412.16720","last_updated":"2026-04-30T02:46:40Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-21T18:04:31Z","title":"OpenAI o1 System Card","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.16720","snapshot_observed_at":"2026-08-07T04:53:38.063638Z","title":"Openai o1 system card","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.063638Z"},"links":{"cited_paper":"/paper/2412.16720","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:e8dbbdc54629102184244db1b1daadd6656c29cedcb8d22379bcafee35c445e4","observation_id":"47d404ba-0f6a-45dc-9ac5-4b27c77f5616","resolution":{"observed_at":"2026-08-07T04:53:38.063638Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.20783","last_updated":"2025-10-06T09:30:03Z","snapshot_observed_at":"2026-08-13T12:34:54.476684Z","submitted_at":"2025-03-26T17:59:14Z","title":"Understanding R1-Zero-Like Training: A Critical Perspective","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.20783","snapshot_observed_at":"2026-08-07T04:53:38.168837Z","title":"Understand- ing r1-zero-like training: A critical perspective","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.168837Z"},"links":{"cited_paper":"/paper/2503.20783","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:cc7196829320ac4de0d46343e5126ef04e8413ee522c810c2526ed77329816fc","observation_id":"12fc0dd7-de8b-408e-bd3c-efb6ce4cf6b6","resolution":{"observed_at":"2026-08-07T04:53:38.168837Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1704.06440","last_updated":"2018-10-14T22:54:38Z","snapshot_observed_at":"2026-08-14T21:05:37.779686Z","submitted_at":"2017-04-21T08:33:59Z","title":"Equivalence Between Policy Gradients and Soft Q-Learning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1704.06440","snapshot_observed_at":"2026-08-07T04:53:38.405144Z","title":"Gradient estimation using stochastic com- putation graphs","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.405144Z"},"links":{"cited_paper":"/paper/1704.06440","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:c2c79820488bacdd5993180b5d70c561ea7bc5ba5dac1e959fc94ad0a440b9b8","observation_id":"fbded47c-c979-4d50-b3c1-d4d2a82b2f49","resolution":{"observed_at":"2026-08-07T04:53:38.405144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.19612","last_updated":"2025-03-28T18:02:54Z","snapshot_observed_at":"2026-08-16T12:47:00.169725Z","submitted_at":"2025-03-25T12:52:38Z","title":"RL-finetuning LLMs from on- and off-policy data with a single algorithm","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.19612","snapshot_observed_at":"2026-08-07T04:53:38.658756Z","title":"Rl-finetuning llms from on-and off- policy data with a single algorithm","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.658756Z"},"links":{"cited_paper":"/paper/2503.19612","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:dcb7c1bfb09e1564fb2e68e40d6308228860bbcbb6814b29ad93d612bfee9a03","observation_id":"f0367503-491f-4ba0-b81e-918e5a46db32","resolution":{"observed_at":"2026-08-07T04:53:38.658756Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.12216","last_updated":"2025-06-03T17:02:25Z","snapshot_observed_at":"2026-08-16T12:32:47.415999Z","submitted_at":"2025-04-16T16:08:45Z","title":"d1: Scaling Reasoning in Diffusion Large Language Models via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.12216","snapshot_observed_at":"2026-08-07T04:53:38.908197Z","title":"d1: Scaling reasoning in diffusion large lan- guage models via reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.908197Z"},"links":{"cited_paper":"/paper/2504.12216","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:edd5feb69c5fc5d73c078956733bede9c20beaa101137ecc866415244ec80c8e","observation_id":"b2518048-a2e4-4371-aa52-f76b644542a7","resolution":{"observed_at":"2026-08-07T04:53:38.908197Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:53:39.977105Z","title":"Details on theoretical results of variance reduced gradient estimate We provide some more details on the theoretical results of the variance reduced gradient estimate","venue":null,"work_id":"41f58392-6407-457e-a254-eedfafe6dd88","year":2004},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:39.096962Z"},"links":{"citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:6cc4408f14783f34c263ffc6d98f7b2d002c66fadbde904988799432ac0859ab","observation_id":"d7fb5e4e-5916-4210-8cf7-b16f66c20057","resolution":{"observed_at":"2026-08-07T04:53:40.066058Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:53:39.785927Z","title":"The reward maximization and on-policy distillation experiments are both carried out on the 7500 MATH training prompts (Hendrycks et al., 2021)","venue":null,"work_id":"9acd3023-0162-49ac-9f9e-882c69f62f16","year":2021},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:39.197075Z"},"links":{"citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:94ee3667c778f505691bc70e68c14fc7eb6ac321a91056a4b7a9b7be17a308c7","observation_id":"bf10f9a3-a2d2-429f-a2f3-747c42b92411","resolution":{"observed_at":"2026-08-07T04:53:39.906857Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:53:38.781775Z","title":"On the design of kl- regularized policy gradient algorithms for llm reasoning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":1992,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.781775Z"},"links":{"citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:400d203dd7bfd340828ffd1f81f414dc045fff255ab9b68972fb098ae0146952","observation_id":"5240904b-dc35-4e2b-924c-87670e4cfafd","resolution":{"observed_at":"2026-08-07T04:53:38.781775Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.05749","last_updated":"2024-05-28T23:25:15Z","snapshot_observed_at":"2026-08-16T14:20:11.482387Z","submitted_at":"2024-02-08T15:33:09Z","title":"Generalized Preference Optimization: A Unified Approach to Offline Alignment","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.05749","snapshot_observed_at":"2026-08-07T04:53:38.578236Z","title":"Generalized preference optimization: A unified approach to offline alignment","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":1999,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.578236Z"},"links":{"cited_paper":"/paper/2402.05749","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:f4ebcc7e6e3aeeae50343884b2d85eea1609830c74574550a1a6c992522ee41d","observation_id":"9fc43729-2309-44a1-b104-6dd6dd3a7ff4","resolution":{"observed_at":"2026-08-07T04:53:38.578236Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-07T04:53:37.534036Z","title":"The llama 3 herd of models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2004,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.534036Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:0c199f6ec76063e45c982a0159c5ca25a348367d6c61b559060e106d2c64766f","observation_id":"47349bc3-35d0-4808-a62a-78d5df0ee905","resolution":{"observed_at":"2026-08-07T04:53:37.534036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.08593","last_updated":"2020-01-08T23:02:36Z","snapshot_observed_at":"2026-08-16T00:15:57.597094Z","submitted_at":"2019-09-18T17:33:39Z","title":"Fine-Tuning Language Models from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.08593","snapshot_observed_at":"2026-08-07T04:53:39.012582Z","title":"Fine-tuning language models from human preferences","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2008,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:39.012582Z"},"links":{"cited_paper":"/paper/1909.08593","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:e94ce59af2ad86f690eb08fcbe9e91086d5f6da33a83cacb135c0bb5fa33cfa8","observation_id":"9fff336c-f0c6-49eb-ae02-0ed116cc2c0c","resolution":{"observed_at":"2026-08-07T04:53:39.012582Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.05453","last_updated":"2025-03-07T14:23:40Z","snapshot_observed_at":"2026-08-16T12:52:09.484090Z","submitted_at":"2025-03-07T14:23:40Z","title":"Soft Policy Optimization: Online Off-Policy RL for Sequence Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.05453","snapshot_observed_at":"2026-08-07T04:53:37.298177Z","title":"Soft policy opti- mization: Online off-policy rl for sequence models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.298177Z"},"links":{"cited_paper":"/paper/2503.05453","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:d21502372477e4ba910ff3a61b1c7bf19799d4d7892a0e5063b51f069edcafb2","observation_id":"1165ba8d-9c0c-44ee-84a0-a7fdd4724392","resolution":{"observed_at":"2026-08-07T04:53:37.298177Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.03874","last_updated":"2021-11-08T21:30:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-03-05T18:59:39Z","title":"Measuring Mathematical Problem Solving With the MATH Dataset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.03874","snapshot_observed_at":"2026-08-07T04:53:37.788104Z","title":"Measuring mathematical problem solving with the math dataset","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.788104Z"},"links":{"cited_paper":"/paper/2103.03874","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:b82af44753233abbb799db538e2a43b31ae27c1ebc006044279b8d652e7704e0","observation_id":"08f1f486-ded2-49d0-ad00-e8831d0d6165","resolution":{"observed_at":"2026-08-07T04:53:37.788104Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1912.01703","last_updated":"2019-12-03T22:06:05Z","snapshot_observed_at":"2026-07-06T08:41:49.632205Z","submitted_at":"2019-12-03T22:06:05Z","title":"PyTorch: An Imperative Style, High-Performance Deep Learning Library","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1912.01703","snapshot_observed_at":"2026-08-07T04:53:38.262205Z","title":null,"venue":null,"work_id":null,"year":1912},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:38.262205Z"},"links":{"cited_paper":"/paper/1912.01703","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:912232256a13359b3c112782f57394d247c368a4ed77b2ed823f1dc153c131df","observation_id":"19b08c8b-4762-4328-8fc2-8a8ff5760855","resolution":{"observed_at":"2026-08-07T04:53:38.262205Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-08-15T12:33:55.451951Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-07T04:53:37.704030Z","title":"Deepseek-r1: Incentivizing reason- ing capability in llms via reinforcement learning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.704030Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:c6cb06eab12aad7710af5f09ebbf03f90a08808b31cd8f62c0b40cd9d807e3f0","observation_id":"65b92a84-cfb6-44a7-8318-60dc0e22414d","resolution":{"observed_at":"2026-08-07T04:53:37.704030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.13399","last_updated":"2025-02-18T17:16:55Z","snapshot_observed_at":"2026-08-16T13:33:06.367631Z","submitted_at":"2024-07-18T11:08:40Z","title":"Correcting the Mythos of KL-Regularization: Direct Alignment without Overoptimization via Chi-Squared Preference Optimization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.13399","snapshot_observed_at":"2026-08-07T04:53:37.946422Z","title":"Correcting the mythos of kl-regularization: Direct align- ment without overoptimization via chi-squared preference optimization","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.946422Z"},"links":{"cited_paper":"/paper/2407.13399","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:d4673eae52fd813f502492a2103b4f338dbc7165cb9a702487e32e86f2a1b7aa","observation_id":"261636fd-99f3-48c9-865a-8eac72226e6d","resolution":{"observed_at":"2026-08-07T04:53:37.946422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.13649","last_updated":"2024-01-17T03:23:23Z","snapshot_observed_at":"2026-08-16T15:21:35.471994Z","submitted_at":"2023-06-23T17:56:26Z","title":"On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.13649","snapshot_observed_at":"2026-08-07T04:53:37.197610Z","title":"Mohammad Gheshlaghi Azar, Zhaohan Daniel Guo, Bilal Piot, Remi Munos, Mark Rowland, Michal Valko, and Daniele Calandriello","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.197610Z"},"links":{"cited_paper":"/paper/2306.13649","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:03922354a35bac6429b71cfacde116c5ff27a8a81ab02fca813fa749e132841e","observation_id":"c7a893ba-dd81-4779-8f18-8dd96e71abbc","resolution":{"observed_at":"2026-08-07T04:53:37.197610Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19185","last_updated":"2025-01-16T10:54:59Z","snapshot_observed_at":"2026-08-16T13:39:04.106583Z","submitted_at":"2024-06-27T14:03:49Z","title":"Contrastive Policy Gradient: Aligning LLMs on sequence-level scores in a supervised-friendly fashion","version":2},"cited_work":{"arxiv_id":"2406.19185","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.19185","snapshot_observed_at":"2026-08-07T04:53:39.544503Z","title":"Contrastive Policy Gradient: Aligning LLMs on sequence-level scores in a supervised-friendly fashion","venue":"cs.LG","work_id":"c002c1a6-46c2-4f61-af5e-f91d3421192d","year":2024},"citing_paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL","version":1},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-07T04:53:37.424674Z"},"links":{"cited_paper":"/paper/2406.19185","citing_paper":"/paper/2506.09477"},"observation_digest":"sha256:7f7869b674e9bc12a6d535f9f6d95f3621f8b4ec6a5b771ed67e7dca75a41ebc","observation_id":"64d1c640-c154-40cd-b440-65865129c217","resolution":{"observed_at":"2026-08-07T04:53:39.648865Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.09477","last_updated":"2025-06-11T07:43:33Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-16T14:29:03.952266Z","submitted_at":"2025-06-11T07:43:33Z","title":"On a few pitfalls in KL divergence gradient estimation for RL"},"reference_resolution":{"displayed":18,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":15,"verified_exact":1,"verified_fuzzy":2},"total_outbound_references":18},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 18 of 18 outbound references and 14 inbound Pith citation observations for arXiv:2506.09477."}