{"as_of":"2026-08-08T15:23:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:1b12e96eb54de58da61a0bfc5393979f6c0fac17dc0d676c73bcf2b00963a3c9","coverage":[{"denominator":46,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":46,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T13:58:44.792769Z","state":"measured"},{"denominator":46,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":46,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.18258/citation-record","integrity":"/paper/2607.18258/integrity","json":"/paper/2607.18258/citation-record.json","paper":"/paper/2607.18258"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:39.777727Z","title":"Deep reinforcement learning from human preferences.Advances in neural information processing systems, 30, 2017","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:39.777727Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:51380db8062b73711739d81f1797ce86a3b7882a14565b98f9eba1365c01ed34","observation_id":"a1d01a30-cc13-41bc-87df-4428c04ff6a0","resolution":{"observed_at":"2026-08-02T13:58:39.777727Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.08593","last_updated":"2020-01-08T23:02:36Z","snapshot_observed_at":"2026-08-08T07:44:49.921146Z","submitted_at":"2019-09-18T17:33:39Z","title":"Fine-Tuning Language Models from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.08593","snapshot_observed_at":"2026-08-02T13:58:39.895334Z","title":"Fine-tuning language models from human preferences","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:39.895334Z"},"links":{"cited_paper":"/paper/1909.08593","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:888901a425009d9b1a223a1335e85b8f84d62d01cea64fe196555d1440608a2b","observation_id":"e997443e-0234-45a4-966f-0222cca11a2f","resolution":{"observed_at":"2026-08-02T13:58:39.895334Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.042083Z","title":"Training language models to follow instructions with human feedback.Advances in neural information processing systems, 35:27730–27744, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.042083Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:037a395f8a4fa42304532f29d17f1fd2e3316cfdd1d19bab7b71a71346296f7f","observation_id":"a641f918-90d0-4565-ac0b-88e965e5dc29","resolution":{"observed_at":"2026-08-02T13:58:40.042083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.181922Z","title":"Rlhf: A comprehensive survey for cultural, multimodal and low latency alignment methods.arXiv preprint arXiv:2511.03939, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.181922Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:b54be89e725f6295280c713335643f88b83bbf3c91d815292e6e6fb7229e442e","observation_id":"9d72c03d-8f2a-4a5f-a77a-4149ff0e054b","resolution":{"observed_at":"2026-08-02T13:58:40.181922Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02743","last_updated":"2025-02-14T23:02:03Z","snapshot_observed_at":"2026-07-06T19:27:11.995338Z","submitted_at":"2024-10-03T17:55:13Z","title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02743","snapshot_observed_at":"2026-08-02T13:58:40.294009Z","title":"Ma-rlhf: Rein- forcement learning from human feedback with macro actions.arXiv preprint arXiv:2410.02743, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.294009Z"},"links":{"cited_paper":"/paper/2410.02743","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:ac7b711ffb59e67213cfd368551e8a09cca2344417e62242b7d6b0a338628493","observation_id":"78a6dedc-8cd2-4cb2-8b7c-7e324c731514","resolution":{"observed_at":"2026-08-02T13:58:40.294009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.437893Z","title":"Confronting reward model overoptimization with constrained RLHF","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.437893Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:b1f6d5e3aa6cf35350140b40ff48612974b18f1d7de69b7ea49ce254b5d2ec01","observation_id":"1d6d387b-8abf-4ccb-9373-7382e95a2bdf","resolution":{"observed_at":"2026-08-02T13:58:40.437893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.554543Z","title":"Regularizing hidden states enables learning generalizable reward model for llms.Advances in Neural Information Processing Systems, 37:62279–62309, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.554543Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:94e284692a493990c75339487a020d1d16a00dd640f8350cb4151161209fa945","observation_id":"2a7161b2-cfe7-4a4f-aa95-d690a487ce79","resolution":{"observed_at":"2026-08-02T13:58:40.554543Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.672240Z","title":"Inform: Mitigating reward hacking in rlhf via information-theoretic reward modeling.Advances in Neural Information Processing Systems, 37:134387–134429, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.672240Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:87279dec3c2c7668484b24246c9cb130ae523952455d1a9b7251676dae8a1827","observation_id":"811c9ace-3bff-4de1-b814-c5edaaea8256","resolution":{"observed_at":"2026-08-02T13:58:40.672240Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.777408Z","title":"DPO meets PPO: Reinforced token optimization for RLHF","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.777408Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:76477794d65414e8529dff6cc29961d19209053bdd62b2b0a447516187874073","observation_id":"9f1947b5-3dd1-4def-8064-62d01c363751","resolution":{"observed_at":"2026-08-02T13:58:40.777408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.883141Z","title":"Information-theoretic reward decomposition for generalizable rlhf.arXiv preprint arXiv:2504.06020, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.883141Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:3bb99f5ba5248c450e84ba7f7a79ef4f0618123fe05acbdf7186cea1357bd250","observation_id":"c691c864-bdc1-4f50-a799-165888626e13","resolution":{"observed_at":"2026-08-02T13:58:40.883141Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:40.971384Z","title":"Capo: Towards enhancing llm reasoning through verifiable generative credit assignment.arXiv e-prints, pages arXiv–2508, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:40.971384Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:1ba393936a34d305447aa38a5be3237a459627166a7ccec0a8816341bd294f9c","observation_id":"466a9e3d-1a39-45d3-9d58-f200606e3a30","resolution":{"observed_at":"2026-08-02T13:58:40.971384Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:41.076306Z","title":"Language models are unsupervised multitask learners.OpenAI blog, 1(8):9, 2019","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.076306Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:a7ac5afc7c2c12e128ae8ae75ddf4e1e427315154d7d5e70a7bb36ba77ddaae8","observation_id":"66bc4ba3-52a9-44f0-8c10-e91d630d0ece","resolution":{"observed_at":"2026-08-02T13:58:41.076306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:41.205872Z","title":"Language models are few-shot learners.Advances in neural information processing systems, 33:1877–1901, 2020","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.205872Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:80c2c50387c8b8206252b14cc5f51de6ca64a909aa89ea4a36bad7a1c63ca310","observation_id":"8b3e7637-bf81-47cb-905e-67f9f28ff4d7","resolution":{"observed_at":"2026-08-02T13:58:41.205872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-02T13:58:41.317159Z","title":"Finetuned language models are zero-shot learners.arXiv preprint arXiv:2109.01652, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.317159Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:e55970beeb7601c0d9b126a0eee50e73fb19154aed65047d7b0b20565de22acb","observation_id":"d68823f1-e86b-47a1-976e-7c63d2205a7d","resolution":{"observed_at":"2026-08-02T13:58:41.317159Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:41.419591Z","title":"Learning to summarize with human feedback","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.419591Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:166aa325badc3c63de384e620d40804420b3f2176c2e50901fb5643b3de833f3","observation_id":"af2e5afc-4658-4ede-8538-ee45c4e8f27b","resolution":{"observed_at":"2026-08-02T13:58:41.419591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04412","last_updated":"2025-03-04T00:04:24Z","snapshot_observed_at":"2026-07-06T18:26:46.402024Z","submitted_at":"2024-06-06T18:01:02Z","title":"Spread Preference Annotation: Direct Preference Judgment for Efficient LLM Alignment","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04412","snapshot_observed_at":"2026-08-02T13:58:41.532409Z","title":"Spread preference annotation: Direct preference judgment for efficient llm alignment.arXiv preprint arXiv:2406.04412, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.532409Z"},"links":{"cited_paper":"/paper/2406.04412","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:bc9d035648a5adc808bc5517650bac57e1d517ccf8148a1080bfa1eddab02b9a","observation_id":"5f21a7eb-3ce9-4467-86ea-e1bea4692410","resolution":{"observed_at":"2026-08-02T13:58:41.532409Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.00626","last_updated":"2025-05-04T15:50:51Z","snapshot_observed_at":"2026-08-07T11:47:24.992650Z","submitted_at":"2022-08-30T02:12:47Z","title":"The Alignment Problem from a Deep Learning Perspective","version":8},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.00626","snapshot_observed_at":"2026-08-02T13:58:41.660778Z","title":"The alignment problem from a deep learning perspective.arXiv preprint arXiv:2209.00626, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.660778Z"},"links":{"cited_paper":"/paper/2209.00626","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:fdd0c139e568dd847a7c1c3cbb4b6d855fd90ce1973ccb03e7f5b5f4b1fd41af","observation_id":"f26ac867-d237-4dfe-8199-f00c18ea6625","resolution":{"observed_at":"2026-08-02T13:58:41.660778Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:41.740345Z","title":"Stepwise alignment for constrained language model policy optimization.Advances in Neural Information Processing Systems, 37:104471–104520, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.740345Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:e7a10d865c25235cd4088f06854718a5a0c7cca8292efb03d3839a7f883e8695","observation_id":"3930e48f-e5d7-44e7-930b-044532b77b35","resolution":{"observed_at":"2026-08-02T13:58:41.740345Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:41.863440Z","title":"Panacea: Pareto alignment via preference adaptation for llms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:41.863440Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:996a0bfc4fa06e94ce954fd9144cb77d4f40bcdec271ec5cc463cd74b29282d1","observation_id":"db1f8bcc-148c-46c9-a36d-ed815553aac2","resolution":{"observed_at":"2026-08-02T13:58:41.863440Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:42.027109Z","title":"Preference learning for AI alignment: a causal perspective","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.027109Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:b207a89d615e64f5155b9ed0e445432b7c8061de8db0093188a9880ccad5b5ca","observation_id":"7bf01e07-aabb-4491-b581-22710502770a","resolution":{"observed_at":"2026-08-02T13:58:42.027109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:42.150952Z","title":"Direct preference optimization: Your language model is secretly a reward model","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.150952Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:b62e99253a277bf0e7c6824640e0fd590ccf512e45d3a0725009676fbede3a05","observation_id":"e0e8ae0d-f299-40f2-8fd8-e7a528581d5c","resolution":{"observed_at":"2026-08-02T13:58:42.150952Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13639","last_updated":"2024-04-30T14:36:26Z","snapshot_observed_at":"2026-08-05T15:32:38.926507Z","submitted_at":"2023-10-20T16:37:56Z","title":"Contrastive Preference Learning: Learning from Human Feedback without RL","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13639","snapshot_observed_at":"2026-08-02T13:58:42.282065Z","title":"Contrastive preference learning: learning from human feedback without rl","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.282065Z"},"links":{"cited_paper":"/paper/2310.13639","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:208a14c686e8d3bf8b6302e20397dbea44c785fe0b7d96277fd945bdf1c79d7a","observation_id":"382bfbfe-7516-4062-ae85-35d2c6448ef3","resolution":{"observed_at":"2026-08-02T13:58:42.282065Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07691","last_updated":"2024-03-14T07:47:08Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-03-12T14:34:08Z","title":"ORPO: Monolithic Preference Optimization without Reference Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07691","snapshot_observed_at":"2026-08-02T13:58:42.386598Z","title":"Orpo: Monolithic preference optimization without reference model.arXiv preprint arXiv:2403.07691, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.386598Z"},"links":{"cited_paper":"/paper/2403.07691","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:3eed50f59a3650e319a45f24b73ce6a825022754db31d53107e8eb2bad7b3f26","observation_id":"79e82d91-7de6-41a1-b1bf-5bfc59681804","resolution":{"observed_at":"2026-08-02T13:58:42.386598Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03300","last_updated":"2024-04-27T15:25:53Z","snapshot_observed_at":"2026-08-06T14:58:42.911363Z","submitted_at":"2024-02-05T18:55:32Z","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03300","snapshot_observed_at":"2026-08-02T13:58:42.521688Z","title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models.arXiv preprint arXiv:2402.03300, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.521688Z"},"links":{"cited_paper":"/paper/2402.03300","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:9fb09328c9f2df444488072e5e6a6cee1db318e71a886adbfaf7a39f5b79f82a","observation_id":"55d6b48d-d7e1-4024-be27-3c0143647f9f","resolution":{"observed_at":"2026-08-02T13:58:42.521688Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:42.664558Z","title":"Scaling laws for reward model overoptimization","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.664558Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:32d2ab3e35d748415b4830b53762a45832b5c822f373a5b344d1d35d78a95bc5","observation_id":"935ed82c-84b8-4118-b846-e028770ca025","resolution":{"observed_at":"2026-08-02T13:58:42.664558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1806.01946","last_updated":"2019-12-23T16:41:02Z","snapshot_observed_at":"2026-08-04T01:49:15.115504Z","submitted_at":"2018-06-05T22:01:51Z","title":"Learning to Understand Goal Specifications by Modelling Reward","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1806.01946","snapshot_observed_at":"2026-08-02T13:58:42.782419Z","title":"Learning to understand goal specifications by modelling reward.arXiv preprint arXiv:1806.01946, 2018","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.782419Z"},"links":{"cited_paper":"/paper/1806.01946","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:9afb5ba3e10229e211195a693744787faff16a2336753c894cf75404f462e2e1","observation_id":"c9812646-da51-41eb-a4e6-fa5dfe955922","resolution":{"observed_at":"2026-08-02T13:58:42.782419Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.02151","last_updated":"2024-05-05T05:04:49Z","snapshot_observed_at":"2026-08-07T03:57:16.625123Z","submitted_at":"2023-08-04T06:14:23Z","title":"Retroformer: Retrospective Large Language Agents with Policy Gradient Optimization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.02151","snapshot_observed_at":"2026-08-02T13:58:42.937172Z","title":"Retroformer: Retrospective large language agents with policy gradient optimization.arXiv preprint arXiv:2308.02151, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.937172Z"},"links":{"cited_paper":"/paper/2308.02151","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:54240dc1f65731444f1a62d37600fa6b5588111591b839d4f25895da37bf8b8e","observation_id":"2d5a709f-7554-4495-9096-3cfa72d7bed2","resolution":{"observed_at":"2026-08-02T13:58:42.937172Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-02T13:58:42.991216Z","title":"Proximal policy optimization algorithms.arXiv preprint arXiv:1707.06347, 2017","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:42.991216Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:63642f1ed1a9b8113d965c6d7b830c400f7e4ed4da1304796a76c76bbe44fd2f","observation_id":"2cc5c481-6322-44df-8ee8-1de5b8809906","resolution":{"observed_at":"2026-08-02T13:58:42.991216Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.16960","last_updated":"2024-05-03T16:30:01Z","snapshot_observed_at":"2026-08-05T00:47:02.714275Z","submitted_at":"2023-10-25T19:58:51Z","title":"Privately Aligning Language Models with Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.16960","snapshot_observed_at":"2026-08-02T13:58:43.111907Z","title":"Privately aligning language models with reinforcement learning.arXiv preprint arXiv:2310.16960, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.111907Z"},"links":{"cited_paper":"/paper/2310.16960","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:eee28984e34109e2b79c02cc6e984b4baefae20aa623f41648cc7c1d47797dfb","observation_id":"f3806584-e0f4-46d1-af92-90f3e81efe65","resolution":{"observed_at":"2026-08-02T13:58:43.111907Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:43.234422Z","title":"Dense reward for free in reinforcement learning from human feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.234422Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:954cb977298268dc4d137f885f990bfb6bc3ba66a9f04964267c74f4ae5ccc0b","observation_id":"960fd674-aabb-4829-8aad-7e28efe7dab5","resolution":{"observed_at":"2026-08-02T13:58:43.234422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:43.317305Z","title":"Tlcr: Token-level continuous reward for fine-grained reinforcement learning from human feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.317305Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:34676ad617ff5e1add43894199074bcd9d93738ea3d6eb314a8de9119d0b79cf","observation_id":"91756c88-c717-4cf3-b916-2d149021361d","resolution":{"observed_at":"2026-08-02T13:58:43.317305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.20417","last_updated":"2025-05-26T18:06:52Z","snapshot_observed_at":"2026-08-07T13:52:46.491642Z","submitted_at":"2025-05-26T18:06:52Z","title":"SCAR: Shapley Credit Assignment for More Efficient RLHF","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.20417","snapshot_observed_at":"2026-08-02T13:58:43.421940Z","title":"Scar: Shapley credit assignment for more efficient rlhf.arXiv preprint arXiv:2505.20417, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.421940Z"},"links":{"cited_paper":"/paper/2505.20417","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:aad413c596de86a03ca0503ffbc4c34e5d17343b22eef5e7af3af31305f862cc","observation_id":"033c1848-ef05-4e5d-80ec-ad830e59491a","resolution":{"observed_at":"2026-08-02T13:58:43.421940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.08302","last_updated":"2025-09-11T10:17:06Z","snapshot_observed_at":"2026-08-05T19:19:51.785740Z","submitted_at":"2024-11-13T02:45:21Z","title":"RED: Unleashing Token-Level Rewards from Holistic Feedback via Reward Redistribution","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.08302","snapshot_observed_at":"2026-08-02T13:58:43.513582Z","title":"R3hf: Reward re- distribution for enhancing reinforcement learning from human feedback.arXiv preprint arXiv:2411.08302, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.513582Z"},"links":{"cited_paper":"/paper/2411.08302","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:a6086047759a7d6d4d0d471a6c00cc4f79cdafbf5f757877af982395c6536fbc","observation_id":"b6b31347-93a8-45a4-91e0-514061b8e5b6","resolution":{"observed_at":"2026-08-02T13:58:43.513582Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:43.614063Z","title":"A critical look at tokenwise reward-guided text generation.arXiv preprint arXiv:2406.07780, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.614063Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:05b0adde33ae3d5151ac2890a22960cd881be528ebb778738f578f44ec688a9c","observation_id":"498542d1-fddb-41dd-8ee0-ee271e8119cc","resolution":{"observed_at":"2026-08-02T13:58:43.614063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:43.712545Z","title":"Judging llm-as-a-judgee with mt-bench and chatbot arena.Advances in neural information processing systems, 36:46595–46623, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.712545Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:89f76175a42cfa9b8192eed13ced835fbb04fceaa31e83f7238897a91510ad22","observation_id":"396a5801-d84e-4043-b36b-9754894356f4","resolution":{"observed_at":"2026-08-02T13:58:43.712545Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21276","last_updated":"2024-10-25T17:43:01Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-10-25T17:43:01Z","title":"GPT-4o System Card","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21276","snapshot_observed_at":"2026-08-02T13:58:43.813439Z","title":"Gpt-4o system card.arXiv preprint arXiv:2410.21276, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.813439Z"},"links":{"cited_paper":"/paper/2410.21276","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:a81bc5f73009b8fa00453438fd28a6ded3203ba0466b8274ea568862a90b0f2c","observation_id":"49ecb033-fee4-43e5-8fe7-f7a3200cdc60","resolution":{"observed_at":"2026-08-02T13:58:43.813439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-02T13:58:43.916499Z","title":"Gpt-4 technical report.arXiv preprint arXiv:2303.08774, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.916499Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:e6d0544850594fef29504e70018d086eebe7e684b044a7949dc45ec3e2fc336d","observation_id":"4c3221b3-a337-4f26-97bb-7572eddb06bc","resolution":{"observed_at":"2026-08-02T13:58:43.916499Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:43.960879Z","title":"Raft: Reward ranked finetuning for generative foundation model alignment.Transactions on Machine Learning Research, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:43.960879Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:0ffa87f413e9923a12fa2896c30d08b90f55b018c2be2bced090e7922f9b4888","observation_id":"8e76b477-3e0a-403b-91ae-c82d6ca1d08c","resolution":{"observed_at":"2026-08-02T13:58:43.960879Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:44.033578Z","title":"Cambridge University Press, 1999","venue":null,"work_id":null,"year":1999},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.033578Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:77eed7980a090ddf7c754975d1904392c6c1c2389e2dcda42a4c5d6d8eb62617","observation_id":"a3cf37e1-420e-40a5-9ee9-a40c56a6fdbf","resolution":{"observed_at":"2026-08-02T13:58:44.033578Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.01017","last_updated":"2022-07-08T12:00:51Z","snapshot_observed_at":"2026-08-03T15:35:05.073940Z","submitted_at":"2022-02-02T13:21:53Z","title":"Multi-Task Learning as a Bargaining Game","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.01017","snapshot_observed_at":"2026-08-02T13:58:44.111762Z","title":"Multi-task learning as a bargaining game.arXiv preprint arXiv:2202.01017, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.111762Z"},"links":{"cited_paper":"/paper/2202.01017","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:3b922f4cfb36ecf4a0413ccc683d28e8682a301eb4bd884b80cb0b9c59dc024d","observation_id":"21c71261-88d4-4fa8-b19b-c396ef0f6bd3","resolution":{"observed_at":"2026-08-02T13:58:44.111762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:58:44.224430Z","title":"Fairness-aware meta-learning via nash bargaining.Advances in Neural Information Processing Systems, 37:83235–83267, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.224430Z"},"links":{"citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:c322176b6f8403c28fca43575347e8046c20c74df28e79478ca44d10c87e33f3","observation_id":"8e27c840-2e87-4f65-a49d-bdf9865982a5","resolution":{"observed_at":"2026-08-02T13:58:44.224430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.11897","last_updated":"2023-08-18T18:35:02Z","snapshot_observed_at":"2026-07-06T15:57:08.701922Z","submitted_at":"2023-07-21T20:54:52Z","title":"Hindsight-DICE: Stable Credit Assignment for Deep Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.11897","snapshot_observed_at":"2026-08-02T13:58:44.322878Z","title":"Hindsight-dice: Stable credit assignment for deep reinforcement learning.arXiv preprint arXiv:2307.11897, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.322878Z"},"links":{"cited_paper":"/paper/2307.11897","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:99d8da966e7cd0d50b4d9606af025e776e56a8b2cc02c74466a86461bd17ac24","observation_id":"c329ffb8-72cb-4586-a3e9-6ff234c50431","resolution":{"observed_at":"2026-08-02T13:58:44.322878Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-02T13:58:44.444288Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.444288Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:f9a9b6bce7e5d11308ea0b594bbc53ac97a2ab11e64e8f60a1dd6373f9f31638","observation_id":"5a37414f-9070-4741-b658-b8279522a888","resolution":{"observed_at":"2026-08-02T13:58:44.444288Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.15043","last_updated":"2023-12-20T20:48:57Z","snapshot_observed_at":"2026-07-06T15:59:23.019044Z","submitted_at":"2023-07-27T17:49:12Z","title":"Universal and Transferable Adversarial Attacks on Aligned Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.15043","snapshot_observed_at":"2026-08-02T13:58:44.543081Z","title":"Universal and transferable adversarial attacks on aligned language models.arXiv preprint arXiv:2307.15043, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.543081Z"},"links":{"cited_paper":"/paper/2307.15043","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:04b28e816d6fc0c4d79c666002a77c6f8b3038626cb75c0b9d6531788eb7b984","observation_id":"230a840b-9267-4b04-b53d-8fdb971f7559","resolution":{"observed_at":"2026-08-02T13:58:44.543081Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.11462","last_updated":"2020-09-25T20:22:26Z","snapshot_observed_at":"2026-07-06T09:58:19.547859Z","submitted_at":"2020-09-24T03:17:19Z","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.11462","snapshot_observed_at":"2026-08-02T13:58:44.661562Z","title":"Real- toxicityprompts: Evaluating neural toxic degeneration in language models.arXiv preprint arXiv:2009.11462, 2020","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.661562Z"},"links":{"cited_paper":"/paper/2009.11462","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:6c05654d347eb4c2f614750dd2fe99d60fb07ab19be8e35ffa84f7c957f58e31","observation_id":"832150d9-4421-4da7-935a-8d3d61499e3a","resolution":{"observed_at":"2026-08-02T13:58:44.661562Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.09509","last_updated":"2022-07-14T13:04:29Z","snapshot_observed_at":"2026-07-06T12:49:18.486644Z","submitted_at":"2022-03-17T17:57:56Z","title":"ToxiGen: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.09509","snapshot_observed_at":"2026-08-02T13:58:44.792769Z","title":"smoother","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-02T13:58:44.792769Z"},"links":{"cited_paper":"/paper/2203.09509","citing_paper":"/paper/2607.18258"},"observation_digest":"sha256:bdfde627efa47218fe37a673110307385ec9ff8667bae30b9284a54070f04f37","observation_id":"b1c598da-52c3-485c-99dc-b42c5ac3be2d","resolution":{"observed_at":"2026-08-02T13:58:44.792769Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.18258","last_updated":"2026-05-15T09:09:57Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-07T01:09:03.575259Z","submitted_at":"2026-05-15T09:09:57Z","title":"S2T-RLHF: Hierarchical Credit Assignment for Stable Preference-Based RLHF"},"reference_resolution":{"displayed":46,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":46,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":46},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 46 of 46 outbound references and 0 inbound Pith citation observations for arXiv:2607.18258."}