{"as_of":"2026-08-16T14:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:4f03fb94d58238b5a961fcc5e1b3bae1f26ca17cee757c47135432f3e7022ba3","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T15:02:45.665708Z","state":"measured"},{"denominator":27,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":27,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-16T06:30:59.297886+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02951/citation-record","integrity":"/paper/2608.02951/integrity","json":"/paper/2608.02951/citation-record.json","paper":"/paper/2608.02951"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-15T15:02:45.558196Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.558196Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:0f8e60f533331a41f821c59804dbd1c45768481cf3e432481a877cc75a1fcbd3","observation_id":"ee422279-496a-4840-9e7d-bfb307e0ec00","resolution":{"observed_at":"2026-08-15T15:02:45.558196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13639","last_updated":"2024-04-30T14:36:26Z","snapshot_observed_at":"2026-08-13T05:44:29.118821Z","submitted_at":"2023-10-20T16:37:56Z","title":"Contrastive Preference Learning: Learning from Human Feedback without RL","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13639","snapshot_observed_at":"2026-08-15T15:02:45.577850Z","title":"Contrastive preference learning: learning from human feedback without rl.arXiv preprint arXiv:2310.13639,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.577850Z"},"links":{"cited_paper":"/paper/2310.13639","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:0f8a558c32f45567b5b6085746a463a26d78f1bc3de73071b70f33addb9136bb","observation_id":"d290f816-01cf-4562-8b67-462bbce1b352","resolution":{"observed_at":"2026-08-15T15:02:45.577850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.15217","last_updated":"2023-09-11T17:25:24Z","snapshot_observed_at":"2026-08-03T19:11:09.671782Z","submitted_at":"2023-07-27T22:29:25Z","title":"Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.15217","snapshot_observed_at":"2026-08-15T15:02:45.582408Z","title":"Open problems and fundamental limitations of reinforcement learning from human feedback.arXiv preprint arXiv:2307.15217,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.582408Z"},"links":{"cited_paper":"/paper/2307.15217","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:6c366f227b83209b1c177d89fd181fd3e9e616e44e9ae06a407e409a48dbf3d1","observation_id":"98b5c4aa-31eb-42e3-a4ba-65e39627fcad","resolution":{"observed_at":"2026-08-15T15:02:45.582408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.11462","last_updated":"2020-09-25T20:22:26Z","snapshot_observed_at":"2026-08-09T05:10:26.091710Z","submitted_at":"2020-09-24T03:17:19Z","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.11462","snapshot_observed_at":"2026-08-15T15:02:45.609451Z","title":"Realtoxicityprompts: Evaluating neural toxic degeneration in language models.arXiv preprint arXiv:2009.11462,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.609451Z"},"links":{"cited_paper":"/paper/2009.11462","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:520f06d2dbc7d5e9bc59a2e2152247f168749f30c46d399d8232ffcb0e38fcb3","observation_id":"3a480b73-c493-4684-92e3-dccca44d5ce0","resolution":{"observed_at":"2026-08-15T15:02:45.609451Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.01876","last_updated":"2025-06-17T17:54:41Z","snapshot_observed_at":"2026-08-09T14:06:21.516916Z","submitted_at":"2025-02-03T23:08:42Z","title":"Reinforcement Learning with Segment Feedback","version":2},"cited_work":{"arxiv_id":"2502.01876","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.01876","snapshot_observed_at":"2026-08-15T15:02:45.745385Z","title":"Reinforcement Learning with Segment Feedback","venue":"cs.LG","work_id":"866c8f08-455b-4b4d-a561-be944139224d","year":2025},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.617996Z"},"links":{"cited_paper":"/paper/2502.01876","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:2d4a3de13d8499d84f5408aa64bf6809da25cfce8ebfeff1270b9c2566a42b46","observation_id":"2f6a9377-259a-45e1-9173-21492ddfe58f","resolution":{"observed_at":"2026-08-15T15:02:45.752291Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6980","last_updated":"2017-01-30T01:27:54Z","snapshot_observed_at":"2026-08-14T18:51:16.666127Z","submitted_at":"2014-12-22T13:54:29Z","title":"Adam: A Method for Stochastic Optimization","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6980","snapshot_observed_at":"2026-08-15T15:02:45.622442Z","title":"Adam: A method for stochastic optimization.arXiv preprint arXiv:1412.6980,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.622442Z"},"links":{"cited_paper":"/paper/1412.6980","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:571f9120146ac36322fe3725ed4fe7bb65a516275d57d986d0bb88e02f593b87","observation_id":"605b043e-97f6-4fc4-89d8-ad25e4b7c7df","resolution":{"observed_at":"2026-08-15T15:02:45.622442Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1711.05101","last_updated":"2019-01-04T21:01:49Z","snapshot_observed_at":"2026-08-14T20:13:52.872565Z","submitted_at":"2017-11-14T14:24:06Z","title":"Decoupled Weight Decay Regularization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1711.05101","snapshot_observed_at":"2026-08-15T15:02:45.631715Z","title":"Decoupled weight decay regularization.arXiv preprint arXiv:1711.05101,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.631715Z"},"links":{"cited_paper":"/paper/1711.05101","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:d915d13cbc67a1be50cae93f892c3c2ef41a882e9d0c1ce4a3352a702e145d26","observation_id":"08c4f306-f481-4970-b9c9-d8c977c836e5","resolution":{"observed_at":"2026-08-15T15:02:45.631715Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.081511Z","title":"Approximating kl divergence, 2020.URL http://joschu","venue":null,"work_id":"9219d6dc-8fe2-4c6f-9468-acbbb41bf6da","year":2020},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.636173Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:26bb21de1c2908637bfd80180c8b9d3172468f2744601a8a6946271ad9614eda","observation_id":"e28fc222-6012-45fa-b84a-03663fb3c60f","resolution":{"observed_at":"2026-08-15T15:02:46.087144Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.066136Z","title":"15 A.2 Comparison of Preference Models","venue":null,"work_id":"fefaa32e-bfb3-4243-afb2-0c6d068c0aa3","year":2011},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.641390Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:d7f063a29ff5405a054ae1c1572fdf324f5f8ac4ab5df7b0cac60672858941fe","observation_id":"4fb484bb-1eb7-4c64-a6d9-305513f72586","resolution":{"observed_at":"2026-08-15T15:02:46.071148Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.050405Z","title":"This makes them inapplicable to many modern RL problems","venue":null,"work_id":"99979dca-aa42-458c-91d9-4ae07314779c","year":2017},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.646471Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:e306ea5a928b16c74fed280b7c751b96c128b7aab32d670f10ca486571dc62f1","observation_id":"ba2a8584-11b2-422a-9c6f-84a761d8d5d0","resolution":{"observed_at":"2026-08-15T15:02:46.055188Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.035761Z","title":"Specifically, even if the partial returns of two segments are the same, 15 humans could still prefer one trajectory segment based on the value of the last state and action","venue":null,"work_id":"5370b888-8067-490b-9063-d02e46db776e","year":2022},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.651001Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:8537953073cafe1680b6498b581f82f2534225d9bbad3dcdc07a9fa717767b8c","observation_id":"13ac6ae2-d619-4207-9ba0-d10702023097","resolution":{"observed_at":"2026-08-15T15:02:46.040674Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.005825Z","title":"During training, policies were Gaussian with a parameter controlling the log standard deviation for each dimension in the action space","venue":null,"work_id":"cc1c36f4-24a2-42ee-b4dc-34f0e3914dcb","year":2022},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.660952Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:27db6100fa2650fee292aaef62249f767b201a0a3ce4ff204c17479511ba029b","observation_id":"156a5eb1-7f77-42e1-b4bb-c7055b8afd38","resolution":{"observed_at":"2026-08-15T15:02:46.010880Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:45.991632Z","title":"The global gradient norm is clipped to1.0","venue":null,"work_id":"9da15089-a36e-4a49-8e6a-86037ce4d528","year":2020},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.665708Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:4e9a224fb7b736c45f962f15d0cfecc34d0ef733230c3c39b0c319f6ce38b0dd","observation_id":"ff72a96f-415a-4d8e-9695-9a1ab3b7e032","resolution":{"observed_at":"2026-08-15T15:02:45.996416Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.020817Z","title":"We observe that in Ant-v5 usingπθt´1 as the second policy out performs setting the second policy toπθref","venue":null,"work_id":"dc801843-3a74-4c18-8bd7-a9e03a56f720","year":2019},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":1000,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.655771Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:aa183ad06022b0c5e07a0331ec986ae2051c75eeae7a9a31ef267c60590952a0","observation_id":"fe301495-77e8-4b26-9e33-1856d6f1db27","resolution":{"observed_at":"2026-08-15T15:02:46.025703Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.02231","last_updated":"2023-09-06T21:13:28Z","snapshot_observed_at":"2026-08-14T15:08:29.695893Z","submitted_at":"2022-06-05T17:58:02Z","title":"Models of human preference for learning reward functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.02231","snapshot_observed_at":"2026-08-15T15:02:45.573168Z","title":"12 Qining Zhang and Lei Ying","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.573168Z"},"links":{"cited_paper":"/paper/2206.02231","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:e9491d44d25e45b73b15a48a03a61a12a3f8c8acab71e295c43633b7f2c42912","observation_id":"34a6412f-ff93-4201-886c-6617e19fa7e0","resolution":{"observed_at":"2026-08-15T15:02:45.573168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.18629","last_updated":"2024-06-26T17:43:06Z","snapshot_observed_at":"2026-08-06T00:24:52.274888Z","submitted_at":"2024-06-26T17:43:06Z","title":"Step-DPO: Step-wise Preference Optimization for Long-chain Reasoning of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.18629","snapshot_observed_at":"2026-08-15T15:02:45.591812Z","title":"Step-dpo: Step-wise preference optimization for long-chain reasoning of llms.arXiv preprint arXiv:2406.18629,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2012,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.591812Z"},"links":{"cited_paper":"/paper/2406.18629","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:39e0627696ac5d493dbec72064633e57203daad63699f171a4ea9588c3da45cc","observation_id":"14ed6ea2-811c-45d0-84b0-51b5553374de","resolution":{"observed_at":"2026-08-15T15:02:45.591812Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-08-15T20:26:32.102285Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-15T15:02:45.548076Z","title":"Proximal policy optimization algorithms.arXiv preprint arXiv:1707.06347,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2013,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.548076Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:b4268581266a0b96cc688be4e08ff18785a38b522593860ded51c7458ea01909","observation_id":"1ff964b8-3fa5-467a-8dfa-88ee91dd45af","resolution":{"observed_at":"2026-08-15T15:02:45.548076Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2208.07587","last_updated":"2022-08-25T06:47:48Z","snapshot_observed_at":"2026-08-16T09:43:43.351907Z","submitted_at":"2022-08-16T08:00:43Z","title":"Making Reinforcement Learning Work on Swimmer","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.07587","snapshot_observed_at":"2026-08-15T15:02:45.626921Z","title":"Making reinforcement learning work on swimmer.arXiv preprint arXiv:2208.07587,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2014,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.626921Z"},"links":{"cited_paper":"/paper/2208.07587","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:01b55bb9dc84392ce877ccf0e2e6b12a02a4eb44b73c60773221950cc85a454d","observation_id":"52bf253f-7a60-4fdd-acb2-db84ff3810f2","resolution":{"observed_at":"2026-08-15T15:02:45.626921Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.05091","last_updated":"2021-06-09T14:10:50Z","snapshot_observed_at":"2026-08-13T19:07:06.766259Z","submitted_at":"2021-06-09T14:10:50Z","title":"PEBBLE: Feedback-Efficient Interactive Reinforcement Learning via Relabeling Experience and Unsupervised Pre-training","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.05091","snapshot_observed_at":"2026-08-15T15:02:45.601173Z","title":"Pebble: Feedback-efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training.arXiv preprint arXiv:2106.05091,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.601173Z"},"links":{"cited_paper":"/paper/2106.05091","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:7e5af111217c18a4052ee065f196d8de7b721dc056cac5e3b38989bcdc287f56","observation_id":"526c189c-caec-415d-a723-96ba1e20aceb","resolution":{"observed_at":"2026-08-15T15:02:45.601173Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.12192","last_updated":"2023-02-23T17:34:53Z","snapshot_observed_at":"2026-07-06T14:55:12.100148Z","submitted_at":"2023-02-23T17:34:53Z","title":"Aligning Text-to-Image Models using Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.12192","snapshot_observed_at":"2026-08-15T15:02:45.567940Z","title":"Aligning text-to-image models using human feedback.arXiv preprint arXiv:2302.12192, 2023b","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.567940Z"},"links":{"cited_paper":"/paper/2302.12192","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:b135b30bdda3e30d222b0242f2b7de894c4db603b76f84e06716d265809a7b1d","observation_id":"179e76b8-8da8-4c1f-8fe0-de3fa777a8c2","resolution":{"observed_at":"2026-08-15T15:02:45.567940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1312.5602","last_updated":"2013-12-19T16:00:08Z","snapshot_observed_at":"2026-08-14T03:19:40.736445Z","submitted_at":"2013-12-19T16:00:08Z","title":"Playing Atari with Deep Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1312.5602","snapshot_observed_at":"2026-08-15T15:02:45.542769Z","title":"Playing atari with deep reinforcement learning.arXiv preprint arXiv:1312.5602,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.542769Z"},"links":{"cited_paper":"/paper/1312.5602","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:7997090ad575b6bc6faa8b8acd519a815ee378e849152fa3c19ca88dc77dede9","observation_id":"b0d7fa27-dac7-474f-9532-0b7e02d9daa7","resolution":{"observed_at":"2026-08-15T15:02:45.542769Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.00957","last_updated":"2023-03-02T04:24:29Z","snapshot_observed_at":"2026-08-13T19:42:31.789282Z","submitted_at":"2023-03-02T04:24:29Z","title":"Preference Transformer: Modeling Human Preferences using Transformers for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.00957","snapshot_observed_at":"2026-08-15T15:02:45.613860Z","title":"Preference transformer: Modeling human preferences using transformers for rl.arXiv preprint arXiv:2303.00957,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.613860Z"},"links":{"cited_paper":"/paper/2303.00957","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:ebf295ce9966e9fac998c44d374b789d77bdd84261b414f0acdb117aad072973","observation_id":"38abb7ee-237b-4218-993a-661c3d4087de","resolution":{"observed_at":"2026-08-15T15:02:45.613860Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.17032","last_updated":"2025-11-02T13:42:19Z","snapshot_observed_at":"2026-08-13T22:24:37.672685Z","submitted_at":"2024-07-24T06:35:05Z","title":"Gymnasium: A Standard Interface for Reinforcement Learning Environments","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.17032","snapshot_observed_at":"2026-08-15T15:02:45.605223Z","title":"Gymnasium: A standard interface for reinforcement learning environments.arXiv preprint arXiv:2407.17032,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.605223Z"},"links":{"cited_paper":"/paper/2407.17032","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:c77c3c5f5b62253cdbf2c796b77e0603b0b9101ded3192b67e023b427e229594","observation_id":"b85b8801-d6b7-4c14-a2b0-d595bc8b65a0","resolution":{"observed_at":"2026-08-15T15:02:45.605223Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.00267","last_updated":"2024-09-03T14:01:54Z","snapshot_observed_at":"2026-08-15T10:07:24.540946Z","submitted_at":"2023-09-01T05:53:33Z","title":"RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.00267","snapshot_observed_at":"2026-08-15T15:02:45.552796Z","title":"Rlaif vs","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.552796Z"},"links":{"cited_paper":"/paper/2309.00267","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:6ccaba8c773b6e36967388bc9d6369a7e6997441f61923c0fe5d882b6e5ad877","observation_id":"ea2ce2c8-9538-4056-8515-e1ea757041a9","resolution":{"observed_at":"2026-08-15T15:02:45.552796Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03300","last_updated":"2024-04-27T15:25:53Z","snapshot_observed_at":"2026-08-06T14:58:42.911363Z","submitted_at":"2024-02-05T18:55:32Z","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03300","snapshot_observed_at":"2026-08-15T15:02:45.563041Z","title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.563041Z"},"links":{"cited_paper":"/paper/2402.03300","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:433968a16ddb4cc6b664405e5b860fcea067768419c45e322f9bc5232bef7dae","observation_id":"227b8e35-4610-4640-bc74-c6d145f81fd4","resolution":{"observed_at":"2026-08-15T15:02:45.563041Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.03066","last_updated":"2026-07-17T19:03:49Z","snapshot_observed_at":"2026-08-12T23:06:28.677740Z","submitted_at":"2025-06-03T16:42:39Z","title":"Sign-SZPO: Provable Preference-based Reinforcement Learning with an Unknown Link Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.03066","snapshot_observed_at":"2026-08-15T15:02:45.587036Z","title":"Provable reinforcement learning from human feedback with an unknown link function","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.587036Z"},"links":{"cited_paper":"/paper/2506.03066","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:89eab526ee9f2fcd135e459be55921e3d051bc2a6cfcb97a53c59a401ef24fe6","observation_id":"f7f7b12a-67c5-4df6-b868-14f03998d776","resolution":{"observed_at":"2026-08-15T15:02:45.587036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.04792","last_updated":"2024-02-29T20:59:17Z","snapshot_observed_at":"2026-08-16T14:20:38.839188Z","submitted_at":"2024-02-07T12:31:13Z","title":"Direct Language Model Alignment from Online AI Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.04792","snapshot_observed_at":"2026-08-15T15:02:45.596461Z","title":"Shangmin Guo, Biao Zhang, Tianlin Liu, Tianqi Liu, Misha Khalman, Felipe Llinares, Alexandre Rame, Thomas Mesnard, Yao Zhao, Bilal Piot, et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2026,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.596461Z"},"links":{"cited_paper":"/paper/2402.04792","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:fb4d6f061934323c7d117d76c185aacf1a4ecf98d636eb2febe4cf42ed99f09d","observation_id":"5d978a87-4205-43fd-bd9a-0b116899fbf5","resolution":{"observed_at":"2026-08-15T15:02:45.596461Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-15T14:52:05.078297Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":19,"verified_exact":1,"verified_fuzzy":7},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"thesis":"As of 16 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 0 inbound Pith citation observations for arXiv:2608.02951."}