{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AABDAKE2UYCVAFT67LLG23SQ63","short_pith_number":"pith:AABDAKE2","canonical_record":{"source":{"id":"2402.10038","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-15T16:00:58Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"0d837eab0b813861e072b65ad3b908b949b6f88179416d61a64f22a168dbde91","abstract_canon_sha256":"5fa59457d9cc8f1cc4b308b52090fb2a8a814e14fbda81641d7d0e5cd1a0fe28"},"schema_version":"1.0"},"canonical_sha256":"000230289aa60550167efad66d6e50f6e1c4717d0b25c7df5792813726293a8d","source":{"kind":"arxiv","id":"2402.10038","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10038","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10038v2","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10038","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_12","alias_value":"AABDAKE2UYCV","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_16","alias_value":"AABDAKE2UYCVAFT6","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_8","alias_value":"AABDAKE2","created_at":"2026-07-05T08:02:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AABDAKE2UYCVAFT67LLG23SQ63","target":"record","payload":{"canonical_record":{"source":{"id":"2402.10038","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-15T16:00:58Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"0d837eab0b813861e072b65ad3b908b949b6f88179416d61a64f22a168dbde91","abstract_canon_sha256":"5fa59457d9cc8f1cc4b308b52090fb2a8a814e14fbda81641d7d0e5cd1a0fe28"},"schema_version":"1.0"},"canonical_sha256":"000230289aa60550167efad66d6e50f6e1c4717d0b25c7df5792813726293a8d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:02:24.727044Z","signature_b64":"zIF/k1OJcA8lwdhzTXlbJ5+g0LbcRRlfM0OM4OmTia7wX0cH6cNZfBKWYojHVq3JPPI6VF3x+/ZjrgteqgByAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"000230289aa60550167efad66d6e50f6e1c4717d0b25c7df5792813726293a8d","last_reissued_at":"2026-07-05T08:02:24.726514Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:02:24.726514Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.10038","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:02:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WZ5OWpk2Nzi1JoPlNlFlMmw7Ffk2Lwa8x67q/hO6MGCY7DydEpsvGPl40gj8l1D/RX/AXt93vi9Ge//aAbiiAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T09:37:30.105295Z"},"content_sha256":"b85323dc4f72dc2d3cc77d2803ec7217f48ce9d209687d21c0bc9a70f78d21f4","schema_version":"1.0","event_id":"sha256:b85323dc4f72dc2d3cc77d2803ec7217f48ce9d209687d21c0bc9a70f78d21f4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AABDAKE2UYCVAFT67LLG23SQ63","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"RS-DPO: A Hybrid Rejection Sampling and Direct Preference Optimization Method for Alignment of Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CV","cs.LG"],"primary_cat":"cs.CL","authors_text":"Jinjin Li, Lan Ma, Liu Yang, Prathap Ramachandra, Saeed Khaki","submitted_at":"2024-02-15T16:00:58Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has been extensively employed to align large language models with user intent. However, proximal policy optimization (PPO) based RLHF is occasionally unstable requiring significant hyperparameter finetuning, and computationally expensive to maximize the estimated reward during alignment. Recently, direct preference optimization (DPO) is proposed to address those challenges. However, DPO relies on contrastive responses generated from human annotator and alternative LLM, instead of the policy model, limiting the effectiveness of the RLHF. In this"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10038","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.10038/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:02:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OBJUEjE0PeZol64KsArzHTgnf8on3JWXlpXNtrPuoMSjCwYZObwQcGo7iojD/qOdVTyrQXDYioMt6ttT4Z3QCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T09:37:30.105762Z"},"content_sha256":"aeb2c5d5cf37e7a6b35deb73bb738bc1b1f433842e7ee6f011c02dd227b753b1","schema_version":"1.0","event_id":"sha256:aeb2c5d5cf37e7a6b35deb73bb738bc1b1f433842e7ee6f011c02dd227b753b1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AABDAKE2UYCVAFT67LLG23SQ63/bundle.json","state_url":"https://pith.science/pith/AABDAKE2UYCVAFT67LLG23SQ63/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AABDAKE2UYCVAFT67LLG23SQ63/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-10T09:37:30Z","links":{"resolver":"https://pith.science/pith/AABDAKE2UYCVAFT67LLG23SQ63","bundle":"https://pith.science/pith/AABDAKE2UYCVAFT67LLG23SQ63/bundle.json","state":"https://pith.science/pith/AABDAKE2UYCVAFT67LLG23SQ63/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AABDAKE2UYCVAFT67LLG23SQ63/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AABDAKE2UYCVAFT67LLG23SQ63","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5fa59457d9cc8f1cc4b308b52090fb2a8a814e14fbda81641d7d0e5cd1a0fe28","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-15T16:00:58Z","title_canon_sha256":"0d837eab0b813861e072b65ad3b908b949b6f88179416d61a64f22a168dbde91"},"schema_version":"1.0","source":{"id":"2402.10038","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10038","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10038v2","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10038","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_12","alias_value":"AABDAKE2UYCV","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_16","alias_value":"AABDAKE2UYCVAFT6","created_at":"2026-07-05T08:02:24Z"},{"alias_kind":"pith_short_8","alias_value":"AABDAKE2","created_at":"2026-07-05T08:02:24Z"}],"graph_snapshots":[{"event_id":"sha256:aeb2c5d5cf37e7a6b35deb73bb738bc1b1f433842e7ee6f011c02dd227b753b1","target":"graph","created_at":"2026-07-05T08:02:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.10038/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has been extensively employed to align large language models with user intent. However, proximal policy optimization (PPO) based RLHF is occasionally unstable requiring significant hyperparameter finetuning, and computationally expensive to maximize the estimated reward during alignment. Recently, direct preference optimization (DPO) is proposed to address those challenges. However, DPO relies on contrastive responses generated from human annotator and alternative LLM, instead of the policy model, limiting the effectiveness of the RLHF. In this","authors_text":"Jinjin Li, Lan Ma, Liu Yang, Prathap Ramachandra, Saeed Khaki","cross_cats":["cs.AI","cs.CV","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-15T16:00:58Z","title":"RS-DPO: A Hybrid Rejection Sampling and Direct Preference Optimization Method for Alignment of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10038","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b85323dc4f72dc2d3cc77d2803ec7217f48ce9d209687d21c0bc9a70f78d21f4","target":"record","created_at":"2026-07-05T08:02:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5fa59457d9cc8f1cc4b308b52090fb2a8a814e14fbda81641d7d0e5cd1a0fe28","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-15T16:00:58Z","title_canon_sha256":"0d837eab0b813861e072b65ad3b908b949b6f88179416d61a64f22a168dbde91"},"schema_version":"1.0","source":{"id":"2402.10038","kind":"arxiv","version":2}},"canonical_sha256":"000230289aa60550167efad66d6e50f6e1c4717d0b25c7df5792813726293a8d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"000230289aa60550167efad66d6e50f6e1c4717d0b25c7df5792813726293a8d","first_computed_at":"2026-07-05T08:02:24.726514Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:02:24.726514Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zIF/k1OJcA8lwdhzTXlbJ5+g0LbcRRlfM0OM4OmTia7wX0cH6cNZfBKWYojHVq3JPPI6VF3x+/ZjrgteqgByAg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:02:24.727044Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.10038","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b85323dc4f72dc2d3cc77d2803ec7217f48ce9d209687d21c0bc9a70f78d21f4","sha256:aeb2c5d5cf37e7a6b35deb73bb738bc1b1f433842e7ee6f011c02dd227b753b1"],"state_sha256":"44dcfc03dc2a729ddd24655d74f90e8cee721d9b1c30eabe9b18ca0e4d09598c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8wy2kN3ccrLbn4rz0bcUog7xK1nArDTDVdpCoaqFZGBSlc9pDBfaaLMCATlwDlmtKakxwTrw/XvSvRMbTyqLCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-10T09:37:30.107747Z","bundle_sha256":"a41734cc96272c7237b46a06396a0cda0098f9b5c5fbc3149088b534cef06e16"}}