{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:ED47JPLPYE7XWMI2AQHRTI2ZQY","short_pith_number":"pith:ED47JPLP","canonical_record":{"source":{"id":"2403.08635","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T15:47:26Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"a53968b4c102eed9f69dbed8ec54c3493188921817cee8b95c7c2b0a8db03c28","abstract_canon_sha256":"78821c512b1de22f458ad2ee179b3ec8b8f6b8e65c3c6a6de5fb12db0fd1a1da"},"schema_version":"1.0"},"canonical_sha256":"20f9f4bd6fc13f7b311a040f19a359863f575cde4bcbaf8ce95a7e34585591cc","source":{"kind":"arxiv","id":"2403.08635","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.08635","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"arxiv_version","alias_value":"2403.08635v1","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.08635","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_12","alias_value":"ED47JPLPYE7X","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_16","alias_value":"ED47JPLPYE7XWMI2","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_8","alias_value":"ED47JPLP","created_at":"2026-07-05T07:55:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:ED47JPLPYE7XWMI2AQHRTI2ZQY","target":"record","payload":{"canonical_record":{"source":{"id":"2403.08635","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T15:47:26Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"a53968b4c102eed9f69dbed8ec54c3493188921817cee8b95c7c2b0a8db03c28","abstract_canon_sha256":"78821c512b1de22f458ad2ee179b3ec8b8f6b8e65c3c6a6de5fb12db0fd1a1da"},"schema_version":"1.0"},"canonical_sha256":"20f9f4bd6fc13f7b311a040f19a359863f575cde4bcbaf8ce95a7e34585591cc","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:55:41.633991Z","signature_b64":"xUjKcdKYwUNsVX01D1d6Lea5vHGf+W5PKlXL1EngWjzjK6LZHhs+VF/0sDDJwaYj0vUDZ673IF4SG2QP3Pd+Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"20f9f4bd6fc13f7b311a040f19a359863f575cde4bcbaf8ce95a7e34585591cc","last_reissued_at":"2026-07-05T07:55:41.633505Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:55:41.633505Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.08635","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"56vAID5YDCWiGRpB79GAr91T9EqsaAC7HD398u2+yfFbMOxcpKo1A0v6yfrIpSObuOof30PtlcsBcRfB1RYBDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:12:18.875604Z"},"content_sha256":"e41d3af4a5509af303e549cadb674bfbf86c0750a305965248a2f29c2eb32129","schema_version":"1.0","event_id":"sha256:e41d3af4a5509af303e549cadb674bfbf86c0750a305965248a2f29c2eb32129"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:ED47JPLPYE7XWMI2AQHRTI2ZQY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Human Alignment of Large Language Models through Online Preference Optimisation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Bernardo Avila Pires, Bilal Piot, Charline Le Lan, Daniele Calandriello, Daniel Guo, Mark Rowland, Michal Valko, Pierre Harvey Richemond, Remi Munos, Rishabh Joshi, Tianqi Liu, Yunhao Tang, Zeyu Zheng","submitted_at":"2024-03-13T15:47:26Z","abstract_excerpt":"Ensuring alignment of language models' outputs with human preferences is critical to guarantee a useful, safe, and pleasant user experience. Thus, human alignment has been extensively studied recently and several methods such as Reinforcement Learning from Human Feedback (RLHF), Direct Policy Optimisation (DPO) and Sequence Likelihood Calibration (SLiC) have emerged. In this paper, our contribution is two-fold. First, we show the equivalence between two recent alignment methods, namely Identity Policy Optimisation (IPO) and Nash Mirror Descent (Nash-MD). Second, we introduce a generalisation o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.08635","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.08635/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D7Kj5GXN2sJkvQbYmVygJShSsMTYl7HOXQeskrBgHb8Bh4FQ4z472mmowM+q/+1F1VOLOIkRRx7Y1Yj9lq/gCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:12:18.876623Z"},"content_sha256":"7d2bb9128e80c5db2d41d0557e13701862c507d740dd684af1d29d9f0d3d2054","schema_version":"1.0","event_id":"sha256:7d2bb9128e80c5db2d41d0557e13701862c507d740dd684af1d29d9f0d3d2054"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/bundle.json","state_url":"https://pith.science/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T13:12:18Z","links":{"resolver":"https://pith.science/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY","bundle":"https://pith.science/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/bundle.json","state":"https://pith.science/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ED47JPLPYE7XWMI2AQHRTI2ZQY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ED47JPLPYE7XWMI2AQHRTI2ZQY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"78821c512b1de22f458ad2ee179b3ec8b8f6b8e65c3c6a6de5fb12db0fd1a1da","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T15:47:26Z","title_canon_sha256":"a53968b4c102eed9f69dbed8ec54c3493188921817cee8b95c7c2b0a8db03c28"},"schema_version":"1.0","source":{"id":"2403.08635","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.08635","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"arxiv_version","alias_value":"2403.08635v1","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.08635","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_12","alias_value":"ED47JPLPYE7X","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_16","alias_value":"ED47JPLPYE7XWMI2","created_at":"2026-07-05T07:55:41Z"},{"alias_kind":"pith_short_8","alias_value":"ED47JPLP","created_at":"2026-07-05T07:55:41Z"}],"graph_snapshots":[{"event_id":"sha256:7d2bb9128e80c5db2d41d0557e13701862c507d740dd684af1d29d9f0d3d2054","target":"graph","created_at":"2026-07-05T07:55:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.08635/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Ensuring alignment of language models' outputs with human preferences is critical to guarantee a useful, safe, and pleasant user experience. Thus, human alignment has been extensively studied recently and several methods such as Reinforcement Learning from Human Feedback (RLHF), Direct Policy Optimisation (DPO) and Sequence Likelihood Calibration (SLiC) have emerged. In this paper, our contribution is two-fold. First, we show the equivalence between two recent alignment methods, namely Identity Policy Optimisation (IPO) and Nash Mirror Descent (Nash-MD). Second, we introduce a generalisation o","authors_text":"Bernardo Avila Pires, Bilal Piot, Charline Le Lan, Daniele Calandriello, Daniel Guo, Mark Rowland, Michal Valko, Pierre Harvey Richemond, Remi Munos, Rishabh Joshi, Tianqi Liu, Yunhao Tang, Zeyu Zheng","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T15:47:26Z","title":"Human Alignment of Large Language Models through Online Preference Optimisation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.08635","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e41d3af4a5509af303e549cadb674bfbf86c0750a305965248a2f29c2eb32129","target":"record","created_at":"2026-07-05T07:55:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"78821c512b1de22f458ad2ee179b3ec8b8f6b8e65c3c6a6de5fb12db0fd1a1da","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T15:47:26Z","title_canon_sha256":"a53968b4c102eed9f69dbed8ec54c3493188921817cee8b95c7c2b0a8db03c28"},"schema_version":"1.0","source":{"id":"2403.08635","kind":"arxiv","version":1}},"canonical_sha256":"20f9f4bd6fc13f7b311a040f19a359863f575cde4bcbaf8ce95a7e34585591cc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"20f9f4bd6fc13f7b311a040f19a359863f575cde4bcbaf8ce95a7e34585591cc","first_computed_at":"2026-07-05T07:55:41.633505Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:55:41.633505Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xUjKcdKYwUNsVX01D1d6Lea5vHGf+W5PKlXL1EngWjzjK6LZHhs+VF/0sDDJwaYj0vUDZ673IF4SG2QP3Pd+Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:55:41.633991Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.08635","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e41d3af4a5509af303e549cadb674bfbf86c0750a305965248a2f29c2eb32129","sha256:7d2bb9128e80c5db2d41d0557e13701862c507d740dd684af1d29d9f0d3d2054"],"state_sha256":"d6bbad3cec396ccdb02754fea52ed9fa4fde486d60b6a83ebae9dbe50416e154"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+1FKpnvIxg3JvI9ZtRiMTfITDLqV2MVute7VjI/CwDCnu+2Ht3F/wrFLPRg8RK+JHApH/gY2fbpBD71SOPjiAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T13:12:18.882728Z","bundle_sha256":"749a1f96ab08783a97be539f3df6fffd89ace352e38c5dfe0bcf88a5e4ed9329"}}