{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:XJ4TZBEW6OFGD77EW4OMMYNUN5","short_pith_number":"pith:XJ4TZBEW","canonical_record":{"source":{"id":"2310.06452","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-10T09:25:44Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"9e9759544ee8ceb86f206016a83e70d3ae4273a792ecf0a4e08bcc752d0d1d40","abstract_canon_sha256":"9588cc882ba242d60e4b2af32935ac29df072e5687afa32cf931a44ea82db0de"},"schema_version":"1.0"},"canonical_sha256":"ba793c8496f38a61ffe4b71cc661b46f71675f7aa75767423913d4da599535aa","source":{"kind":"arxiv","id":"2310.06452","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.06452","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"arxiv_version","alias_value":"2310.06452v3","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.06452","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_12","alias_value":"XJ4TZBEW6OFG","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_16","alias_value":"XJ4TZBEW6OFGD77E","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_8","alias_value":"XJ4TZBEW","created_at":"2026-05-19T02:28:36Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:XJ4TZBEW6OFGD77EW4OMMYNUN5","target":"record","payload":{"canonical_record":{"source":{"id":"2310.06452","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-10T09:25:44Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"9e9759544ee8ceb86f206016a83e70d3ae4273a792ecf0a4e08bcc752d0d1d40","abstract_canon_sha256":"9588cc882ba242d60e4b2af32935ac29df072e5687afa32cf931a44ea82db0de"},"schema_version":"1.0"},"canonical_sha256":"ba793c8496f38a61ffe4b71cc661b46f71675f7aa75767423913d4da599535aa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-19T02:28:36.403682Z","signature_b64":"01PTiZ9BUtWJmj19Hrf1w+YnTMfsIzF16VTFaZwqhUahtj9PglbRc0sCv0E1vbEr8xBrM3H8J02d2T8oyno4Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ba793c8496f38a61ffe4b71cc661b46f71675f7aa75767423913d4da599535aa","last_reissued_at":"2026-05-19T02:28:36.401174Z","signature_status":"signed_v1","first_computed_at":"2026-05-19T02:28:36.401174Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.06452","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-19T02:28:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JcIb07s5wnNhJdEUslVVqDjtEBSUxG/p3cexkfWrJbjMItRlteEQb0F+DRexK0KVrD7S+gCeb4CPntGgI8E1DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-19T23:23:51.778865Z"},"content_sha256":"3575a86093e00f4f35369a5e39fc55be91204213aa828bce97556886eb75577f","schema_version":"1.0","event_id":"sha256:3575a86093e00f4f35369a5e39fc55be91204213aa828bce97556886eb75577f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:XJ4TZBEW6OFGD77EW4OMMYNUN5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Understanding the Effects of RLHF on LLM Generalisation and Diversity","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Christoforos Nalmpantis, Edward Grefenstette, Eric Hambro, Ishita Mediratta, Jelena Luketina, Roberta Raileanu, Robert Kirk","submitted_at":"2023-10-10T09:25:44Z","abstract_excerpt":"Large language models (LLMs) fine-tuned with reinforcement learning from human feedback (RLHF) have been used in some of the most widely deployed AI models to date, such as OpenAI's ChatGPT or Anthropic's Claude. While there has been significant work developing these methods, our understanding of the benefits and downsides of each stage in RLHF is still limited. To fill this gap, we present an extensive analysis of how each stage of the process (i.e. supervised fine-tuning (SFT), reward modelling, and RLHF) affects two key properties: out-of-distribution (OOD) generalisation and output diversi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.06452","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-19T02:28:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I6/W0rHH6W1e+BvPWdjbv/H7yXlC2YmCTHa20AOydOJX01njabV8R38khX6ahqFk/Hm2vcGxcopxUMyjvy78Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-19T23:23:51.779284Z"},"content_sha256":"32c7671cffef4dcf17941a9e6e21a84d598412d0aea7a041e55baddebda68d7e","schema_version":"1.0","event_id":"sha256:32c7671cffef4dcf17941a9e6e21a84d598412d0aea7a041e55baddebda68d7e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/bundle.json","state_url":"https://pith.science/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-19T23:23:51Z","links":{"resolver":"https://pith.science/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5","bundle":"https://pith.science/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/bundle.json","state":"https://pith.science/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XJ4TZBEW6OFGD77EW4OMMYNUN5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:XJ4TZBEW6OFGD77EW4OMMYNUN5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9588cc882ba242d60e4b2af32935ac29df072e5687afa32cf931a44ea82db0de","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-10T09:25:44Z","title_canon_sha256":"9e9759544ee8ceb86f206016a83e70d3ae4273a792ecf0a4e08bcc752d0d1d40"},"schema_version":"1.0","source":{"id":"2310.06452","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.06452","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"arxiv_version","alias_value":"2310.06452v3","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.06452","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_12","alias_value":"XJ4TZBEW6OFG","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_16","alias_value":"XJ4TZBEW6OFGD77E","created_at":"2026-05-19T02:28:36Z"},{"alias_kind":"pith_short_8","alias_value":"XJ4TZBEW","created_at":"2026-05-19T02:28:36Z"}],"graph_snapshots":[{"event_id":"sha256:32c7671cffef4dcf17941a9e6e21a84d598412d0aea7a041e55baddebda68d7e","target":"graph","created_at":"2026-05-19T02:28:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Large language models (LLMs) fine-tuned with reinforcement learning from human feedback (RLHF) have been used in some of the most widely deployed AI models to date, such as OpenAI's ChatGPT or Anthropic's Claude. While there has been significant work developing these methods, our understanding of the benefits and downsides of each stage in RLHF is still limited. To fill this gap, we present an extensive analysis of how each stage of the process (i.e. supervised fine-tuning (SFT), reward modelling, and RLHF) affects two key properties: out-of-distribution (OOD) generalisation and output diversi","authors_text":"Christoforos Nalmpantis, Edward Grefenstette, Eric Hambro, Ishita Mediratta, Jelena Luketina, Roberta Raileanu, Robert Kirk","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-10T09:25:44Z","title":"Understanding the Effects of RLHF on LLM Generalisation and Diversity"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.06452","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3575a86093e00f4f35369a5e39fc55be91204213aa828bce97556886eb75577f","target":"record","created_at":"2026-05-19T02:28:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9588cc882ba242d60e4b2af32935ac29df072e5687afa32cf931a44ea82db0de","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-10-10T09:25:44Z","title_canon_sha256":"9e9759544ee8ceb86f206016a83e70d3ae4273a792ecf0a4e08bcc752d0d1d40"},"schema_version":"1.0","source":{"id":"2310.06452","kind":"arxiv","version":3}},"canonical_sha256":"ba793c8496f38a61ffe4b71cc661b46f71675f7aa75767423913d4da599535aa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ba793c8496f38a61ffe4b71cc661b46f71675f7aa75767423913d4da599535aa","first_computed_at":"2026-05-19T02:28:36.401174Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-19T02:28:36.401174Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"01PTiZ9BUtWJmj19Hrf1w+YnTMfsIzF16VTFaZwqhUahtj9PglbRc0sCv0E1vbEr8xBrM3H8J02d2T8oyno4Cw==","signature_status":"signed_v1","signed_at":"2026-05-19T02:28:36.403682Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.06452","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3575a86093e00f4f35369a5e39fc55be91204213aa828bce97556886eb75577f","sha256:32c7671cffef4dcf17941a9e6e21a84d598412d0aea7a041e55baddebda68d7e"],"state_sha256":"60b5048b8f65dc95d0ed5cf9ac7eba3a6afc57b4ad9cd1a3321547a8e5fbabd2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VXupPTcGI+R1MfJZ+fgIyYyshSXVWxG5q1Ad1MbY7wCg5uP19nfWyAT/n126qPncNGpjtAkd8TZbscp0C7vxCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-19T23:23:51.782930Z","bundle_sha256":"cfda331342c5e6ea97d9404ec0d05e7396eb39d8c53d35826352cdb64ae204e5"}}