{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2TAQZXOZH2HNYCTC74RYD7GBVB","short_pith_number":"pith:2TAQZXOZ","canonical_record":{"source":{"id":"2410.18252","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"bf0402ab32c496401c009c0bcec4287787a0ec7d8a061d30967554184ef0ca05","abstract_canon_sha256":"df25b02154dc0a1279acf9171edfdfbe188d5b69b5a5cf99a70837cdbd5c1277"},"schema_version":"1.0"},"canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","source":{"kind":"arxiv","id":"2410.18252","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.18252","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"arxiv_version","alias_value":"2410.18252v3","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.18252","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_12","alias_value":"2TAQZXOZH2HN","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_16","alias_value":"2TAQZXOZH2HNYCTC","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_8","alias_value":"2TAQZXOZ","created_at":"2026-07-05T10:54:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2TAQZXOZH2HNYCTC74RYD7GBVB","target":"record","payload":{"canonical_record":{"source":{"id":"2410.18252","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"bf0402ab32c496401c009c0bcec4287787a0ec7d8a061d30967554184ef0ca05","abstract_canon_sha256":"df25b02154dc0a1279acf9171edfdfbe188d5b69b5a5cf99a70837cdbd5c1277"},"schema_version":"1.0"},"canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:13.559705Z","signature_b64":"zEJYMJ30+nrLbvxsYbYAUT25durOvtapNiwkxMJwEriWE34Yw/ENFjWsc7ao2fPLfKrP2y5qT7vli1T/csKVDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","last_reissued_at":"2026-07-05T10:54:13.559190Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:13.559190Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.18252","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NjNNDuoMtgBw/kAuABYQ1yMbRat8cqVmildSJU/GB0BTt/cpuPAmLz82ii8geZG10LR+xg+wBYkMSToFPTPzCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T09:57:13.213336Z"},"content_sha256":"1d73adbf295a5a90338fba03a0609c0686f2d9114ab5bb7640c88ceade08c654","schema_version":"1.0","event_id":"sha256:1d73adbf295a5a90338fba03a0609c0686f2d9114ab5bb7640c88ceade08c654"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2TAQZXOZH2HNYCTC74RYD7GBVB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Aaron Courville, Arian Hosseini, Michael Noukhovitch, Rishabh Agarwal, Shengyi Huang, Sophie Xhonneux","submitted_at":"2024-10-23T19:59:50Z","abstract_excerpt":"The dominant paradigm for RLHF is online and on-policy RL: synchronously generating from the large language model (LLM) policy, labelling with a reward model, and learning using feedback on the LLM's own outputs. While performant, this paradigm is computationally inefficient. Inspired by classical deep RL literature, we propose separating generation and learning in RLHF. This enables asynchronous generation of new samples while simultaneously training on old samples, leading to faster training and more compute-optimal scaling. However, asynchronous training relies on an underexplored regime, o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.18252","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.18252/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aXnl21NXWSric26S0UJcDT+yLst76500FWNqy3wnCLYDnyRUB+abhhrUibz7W5ETfizj+NwEUcqUcafy7MHmDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T09:57:13.213718Z"},"content_sha256":"16128a7a48233fe6a8902fbca095bffdd6696630a48fe69daff9473e53539084","schema_version":"1.0","event_id":"sha256:16128a7a48233fe6a8902fbca095bffdd6696630a48fe69daff9473e53539084"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/bundle.json","state_url":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T09:57:13Z","links":{"resolver":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB","bundle":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/bundle.json","state":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2TAQZXOZH2HNYCTC74RYD7GBVB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"df25b02154dc0a1279acf9171edfdfbe188d5b69b5a5cf99a70837cdbd5c1277","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","title_canon_sha256":"bf0402ab32c496401c009c0bcec4287787a0ec7d8a061d30967554184ef0ca05"},"schema_version":"1.0","source":{"id":"2410.18252","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.18252","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"arxiv_version","alias_value":"2410.18252v3","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.18252","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_12","alias_value":"2TAQZXOZH2HN","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_16","alias_value":"2TAQZXOZH2HNYCTC","created_at":"2026-07-05T10:54:13Z"},{"alias_kind":"pith_short_8","alias_value":"2TAQZXOZ","created_at":"2026-07-05T10:54:13Z"}],"graph_snapshots":[{"event_id":"sha256:16128a7a48233fe6a8902fbca095bffdd6696630a48fe69daff9473e53539084","target":"graph","created_at":"2026-07-05T10:54:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.18252/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The dominant paradigm for RLHF is online and on-policy RL: synchronously generating from the large language model (LLM) policy, labelling with a reward model, and learning using feedback on the LLM's own outputs. While performant, this paradigm is computationally inefficient. Inspired by classical deep RL literature, we propose separating generation and learning in RLHF. This enables asynchronous generation of new samples while simultaneously training on old samples, leading to faster training and more compute-optimal scaling. However, asynchronous training relies on an underexplored regime, o","authors_text":"Aaron Courville, Arian Hosseini, Michael Noukhovitch, Rishabh Agarwal, Shengyi Huang, Sophie Xhonneux","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","title":"Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.18252","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1d73adbf295a5a90338fba03a0609c0686f2d9114ab5bb7640c88ceade08c654","target":"record","created_at":"2026-07-05T10:54:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"df25b02154dc0a1279acf9171edfdfbe188d5b69b5a5cf99a70837cdbd5c1277","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","title_canon_sha256":"bf0402ab32c496401c009c0bcec4287787a0ec7d8a061d30967554184ef0ca05"},"schema_version":"1.0","source":{"id":"2410.18252","kind":"arxiv","version":3}},"canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","first_computed_at":"2026-07-05T10:54:13.559190Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:54:13.559190Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zEJYMJ30+nrLbvxsYbYAUT25durOvtapNiwkxMJwEriWE34Yw/ENFjWsc7ao2fPLfKrP2y5qT7vli1T/csKVDA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:54:13.559705Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.18252","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1d73adbf295a5a90338fba03a0609c0686f2d9114ab5bb7640c88ceade08c654","sha256:16128a7a48233fe6a8902fbca095bffdd6696630a48fe69daff9473e53539084"],"state_sha256":"ae1fa840310f203687d85321a4e546d0576caf1af83c1bbd9e89111a42116882"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8qc59SbZr6GATG6SPlBJksyEQYDKRilbBbxfDf2gxocrQy9Jq8+S79jU3s2K/C0m4qTl/zuOtMgemPmmENz1Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T09:57:13.215763Z","bundle_sha256":"37ebb0d184a660d4b88631e3351a8c308c9b6b703e100046ab4acff59fe555e7"}}