{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:3QOXZ4BQBXZEZSFX25YRBQ7KKR","short_pith_number":"pith:3QOXZ4BQ","canonical_record":{"source":{"id":"2405.19107","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T14:11:29Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a6331ee315f03d7d4b7c88e28046509d55bd2c95b0aec8245c064ba245fd3681","abstract_canon_sha256":"004025d79f6453aaab78db2e6e36fe5bacf1cf4a43185844e3b4539dd4c28853"},"schema_version":"1.0"},"canonical_sha256":"dc1d7cf0300df24cc8b7d77110c3ea547af089f65b0a5b9c677693a7a59262b0","source":{"kind":"arxiv","id":"2405.19107","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19107","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19107v1","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19107","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_12","alias_value":"3QOXZ4BQBXZE","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_16","alias_value":"3QOXZ4BQBXZEZSFX","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_8","alias_value":"3QOXZ4BQ","created_at":"2026-07-05T08:24:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:3QOXZ4BQBXZEZSFX25YRBQ7KKR","target":"record","payload":{"canonical_record":{"source":{"id":"2405.19107","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T14:11:29Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a6331ee315f03d7d4b7c88e28046509d55bd2c95b0aec8245c064ba245fd3681","abstract_canon_sha256":"004025d79f6453aaab78db2e6e36fe5bacf1cf4a43185844e3b4539dd4c28853"},"schema_version":"1.0"},"canonical_sha256":"dc1d7cf0300df24cc8b7d77110c3ea547af089f65b0a5b9c677693a7a59262b0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:24:47.994557Z","signature_b64":"9Wre/ZIkHjIK51Vc/jHeYspMcuunHNj3AIDNe1Qglhq6d1w4sbeYl0LpMmAJ19rc6Ta+Cj2YIo78AJGu5m5ZAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dc1d7cf0300df24cc8b7d77110c3ea547af089f65b0a5b9c677693a7a59262b0","last_reissued_at":"2026-07-05T08:24:47.994059Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:24:47.994059Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.19107","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"X8dcTTmY/21s0fYKIEgdr2GM2XXt5J4iXUQGdgNPOaAqPtawejeHHY1hczpxMYRZRS14oyHU3iMlPcgLJP2kDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T20:47:48.459106Z"},"content_sha256":"ca696bf331167f4289c799f48e94a0fb1fd33b377f365b101056fa17b78a7d27","schema_version":"1.0","event_id":"sha256:ca696bf331167f4289c799f48e94a0fb1fd33b377f365b101056fa17b78a7d27"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:3QOXZ4BQBXZEZSFX25YRBQ7KKR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Offline Regularised Reinforcement Learning for Large Language Models Alignment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Aliaksei Severyn, Bernardo Avila Pires, Bilal Piot, Daniele Calandriello, Daniel Guo, Eugene Tarassov, Gil Shamir, Jonathan Mallinson, Lior Shani, Lucas Spangher, Mohammad Gheshlaghi Azar, Pierre Harvey Richemond, Rafael Rafailov, Remi Munos, Rishabh Joshi, Tianqi Liu, Will Ellsworth, Yunhao Tang","submitted_at":"2024-05-29T14:11:29Z","abstract_excerpt":"The dominant framework for alignment of large language models (LLM), whether through reinforcement learning from human feedback or direct preference optimisation, is to learn from preference data. This involves building datasets where each element is a quadruplet composed of a prompt, two independent responses (completions of the prompt) and a human preference between the two independent responses, yielding a preferred and a dis-preferred response. Such data is typically scarce and expensive to collect. On the other hand, \\emph{single-trajectory} datasets where each element is a triplet compos"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19107","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.19107/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iwohTwMexVHbHUFhTVhDDqpnUEqR80mwam13Id1oNyTvhnuIe1GHrhh0Eqtaycm14D9SR79oS+4Fw0hs+tbmAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T20:47:48.459659Z"},"content_sha256":"9d917280591dbd5a8d8d7667feb5da6391abbc1c3ae401753fa5750331eb20aa","schema_version":"1.0","event_id":"sha256:9d917280591dbd5a8d8d7667feb5da6391abbc1c3ae401753fa5750331eb20aa"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/bundle.json","state_url":"https://pith.science/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-19T20:47:48Z","links":{"resolver":"https://pith.science/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR","bundle":"https://pith.science/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/bundle.json","state":"https://pith.science/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3QOXZ4BQBXZEZSFX25YRBQ7KKR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3QOXZ4BQBXZEZSFX25YRBQ7KKR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"004025d79f6453aaab78db2e6e36fe5bacf1cf4a43185844e3b4539dd4c28853","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T14:11:29Z","title_canon_sha256":"a6331ee315f03d7d4b7c88e28046509d55bd2c95b0aec8245c064ba245fd3681"},"schema_version":"1.0","source":{"id":"2405.19107","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19107","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19107v1","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19107","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_12","alias_value":"3QOXZ4BQBXZE","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_16","alias_value":"3QOXZ4BQBXZEZSFX","created_at":"2026-07-05T08:24:47Z"},{"alias_kind":"pith_short_8","alias_value":"3QOXZ4BQ","created_at":"2026-07-05T08:24:47Z"}],"graph_snapshots":[{"event_id":"sha256:9d917280591dbd5a8d8d7667feb5da6391abbc1c3ae401753fa5750331eb20aa","target":"graph","created_at":"2026-07-05T08:24:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.19107/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The dominant framework for alignment of large language models (LLM), whether through reinforcement learning from human feedback or direct preference optimisation, is to learn from preference data. This involves building datasets where each element is a quadruplet composed of a prompt, two independent responses (completions of the prompt) and a human preference between the two independent responses, yielding a preferred and a dis-preferred response. Such data is typically scarce and expensive to collect. On the other hand, \\emph{single-trajectory} datasets where each element is a triplet compos","authors_text":"Aliaksei Severyn, Bernardo Avila Pires, Bilal Piot, Daniele Calandriello, Daniel Guo, Eugene Tarassov, Gil Shamir, Jonathan Mallinson, Lior Shani, Lucas Spangher, Mohammad Gheshlaghi Azar, Pierre Harvey Richemond, Rafael Rafailov, Remi Munos, Rishabh Joshi, Tianqi Liu, Will Ellsworth, Yunhao Tang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T14:11:29Z","title":"Offline Regularised Reinforcement Learning for Large Language Models Alignment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19107","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ca696bf331167f4289c799f48e94a0fb1fd33b377f365b101056fa17b78a7d27","target":"record","created_at":"2026-07-05T08:24:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"004025d79f6453aaab78db2e6e36fe5bacf1cf4a43185844e3b4539dd4c28853","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T14:11:29Z","title_canon_sha256":"a6331ee315f03d7d4b7c88e28046509d55bd2c95b0aec8245c064ba245fd3681"},"schema_version":"1.0","source":{"id":"2405.19107","kind":"arxiv","version":1}},"canonical_sha256":"dc1d7cf0300df24cc8b7d77110c3ea547af089f65b0a5b9c677693a7a59262b0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dc1d7cf0300df24cc8b7d77110c3ea547af089f65b0a5b9c677693a7a59262b0","first_computed_at":"2026-07-05T08:24:47.994059Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:24:47.994059Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9Wre/ZIkHjIK51Vc/jHeYspMcuunHNj3AIDNe1Qglhq6d1w4sbeYl0LpMmAJ19rc6Ta+Cj2YIo78AJGu5m5ZAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:24:47.994557Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.19107","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ca696bf331167f4289c799f48e94a0fb1fd33b377f365b101056fa17b78a7d27","sha256:9d917280591dbd5a8d8d7667feb5da6391abbc1c3ae401753fa5750331eb20aa"],"state_sha256":"c99dda308cdc93f319db07f49aa290c1d2b6133e063695179e78f1132940668a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"55TDzp5MHYceJQBIxk9wnwnpzCXR7FJterla0Xncdsi2Z2YUPptp/99udKyZKDYtdX0T6TfnJvTNO05lt/VeDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-19T20:47:48.463534Z","bundle_sha256":"d4f23b3589a09e31e52c1313cc589fcf197df59e14e5416f8df045cefdd741f8"}}