{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:ZNTNUA2ZYABWDWWYIBEZWK3W3M","short_pith_number":"pith:ZNTNUA2Z","canonical_record":{"source":{"id":"2402.08005","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-12T19:10:13Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"322a6139774c34e13d5e7ec6f01a8b95adff823a9bf91db974d574124e85bf72","abstract_canon_sha256":"d2958e7d20a3a1af4094bee8e004de335d737d370eea92883093bdf3ca9dbec3"},"schema_version":"1.0"},"canonical_sha256":"cb66da0359c00361dad840499b2b76db06b9e7f10dd4a88de813c5b55cb04f8f","source":{"kind":"arxiv","id":"2402.08005","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.08005","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"arxiv_version","alias_value":"2402.08005v1","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.08005","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_12","alias_value":"ZNTNUA2ZYABW","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_16","alias_value":"ZNTNUA2ZYABWDWWY","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_8","alias_value":"ZNTNUA2Z","created_at":"2026-07-05T07:44:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:ZNTNUA2ZYABWDWWYIBEZWK3W3M","target":"record","payload":{"canonical_record":{"source":{"id":"2402.08005","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-12T19:10:13Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"322a6139774c34e13d5e7ec6f01a8b95adff823a9bf91db974d574124e85bf72","abstract_canon_sha256":"d2958e7d20a3a1af4094bee8e004de335d737d370eea92883093bdf3ca9dbec3"},"schema_version":"1.0"},"canonical_sha256":"cb66da0359c00361dad840499b2b76db06b9e7f10dd4a88de813c5b55cb04f8f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:44:25.634958Z","signature_b64":"pTdCjlYUzqByeAbFZupE8iSX1mqjPS0cKfGxxK1z6XzhFXW8FEZNcYNkq+2RynsH8rZnKbwzM+Ru0S5XfdqFDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cb66da0359c00361dad840499b2b76db06b9e7f10dd4a88de813c5b55cb04f8f","last_reissued_at":"2026-07-05T07:44:25.634466Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:44:25.634466Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.08005","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:44:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nHam0pDpIojqUzA2NyVR7kaBlMuoNPvuxVWjF7lY6y6POgXepqBstHw52bFgsLV4gCBU7GTigLlxWh4L53QYAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T13:50:20.262785Z"},"content_sha256":"11a4fd10ac8ba90955a808a2033569e57a7230b9b31a5b430139b1f77b749878","schema_version":"1.0","event_id":"sha256:11a4fd10ac8ba90955a808a2033569e57a7230b9b31a5b430139b1f77b749878"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:ZNTNUA2ZYABWDWWYIBEZWK3W3M","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Refined Direct Preference Optimization with Synthetic Data for Behavioral Alignment of LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"V\\'ictor Gallego","submitted_at":"2024-02-12T19:10:13Z","abstract_excerpt":"In this paper, we introduce \\emph{refined Direct Preference Optimization} (rDPO), a method for improving the behavioral alignment of Large Language Models (LLMs) without the need for human-annotated data. The method involves creating synthetic data using self-critique prompting by a teacher LLM and then utilising a generalized DPO loss function to distil to a student LLM. The loss function incorporates an additional external reward model to improve the quality of synthetic data, making rDPO robust to potential noise in the synthetic dataset. rDPO is shown to be effective in a diverse set of be"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.08005","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.08005/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:44:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jv18U+L8+JMF0HA68kiweMnM2mBLrMnvxS3wHLZMjEpM7PLhu/ERBBFpYTgNgbJpuknScS+e7g0OaV+UZDy7Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T13:50:20.263336Z"},"content_sha256":"073f632a91b698ab5b789dbfd2a8a0a18e2602476802f422d1e37d3032bdb144","schema_version":"1.0","event_id":"sha256:073f632a91b698ab5b789dbfd2a8a0a18e2602476802f422d1e37d3032bdb144"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/bundle.json","state_url":"https://pith.science/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T13:50:20Z","links":{"resolver":"https://pith.science/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M","bundle":"https://pith.science/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/bundle.json","state":"https://pith.science/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZNTNUA2ZYABWDWWYIBEZWK3W3M/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ZNTNUA2ZYABWDWWYIBEZWK3W3M","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d2958e7d20a3a1af4094bee8e004de335d737d370eea92883093bdf3ca9dbec3","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-12T19:10:13Z","title_canon_sha256":"322a6139774c34e13d5e7ec6f01a8b95adff823a9bf91db974d574124e85bf72"},"schema_version":"1.0","source":{"id":"2402.08005","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.08005","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"arxiv_version","alias_value":"2402.08005v1","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.08005","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_12","alias_value":"ZNTNUA2ZYABW","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_16","alias_value":"ZNTNUA2ZYABWDWWY","created_at":"2026-07-05T07:44:25Z"},{"alias_kind":"pith_short_8","alias_value":"ZNTNUA2Z","created_at":"2026-07-05T07:44:25Z"}],"graph_snapshots":[{"event_id":"sha256:073f632a91b698ab5b789dbfd2a8a0a18e2602476802f422d1e37d3032bdb144","target":"graph","created_at":"2026-07-05T07:44:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.08005/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we introduce \\emph{refined Direct Preference Optimization} (rDPO), a method for improving the behavioral alignment of Large Language Models (LLMs) without the need for human-annotated data. The method involves creating synthetic data using self-critique prompting by a teacher LLM and then utilising a generalized DPO loss function to distil to a student LLM. The loss function incorporates an additional external reward model to improve the quality of synthetic data, making rDPO robust to potential noise in the synthetic dataset. rDPO is shown to be effective in a diverse set of be","authors_text":"V\\'ictor Gallego","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-12T19:10:13Z","title":"Refined Direct Preference Optimization with Synthetic Data for Behavioral Alignment of LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.08005","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:11a4fd10ac8ba90955a808a2033569e57a7230b9b31a5b430139b1f77b749878","target":"record","created_at":"2026-07-05T07:44:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d2958e7d20a3a1af4094bee8e004de335d737d370eea92883093bdf3ca9dbec3","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-12T19:10:13Z","title_canon_sha256":"322a6139774c34e13d5e7ec6f01a8b95adff823a9bf91db974d574124e85bf72"},"schema_version":"1.0","source":{"id":"2402.08005","kind":"arxiv","version":1}},"canonical_sha256":"cb66da0359c00361dad840499b2b76db06b9e7f10dd4a88de813c5b55cb04f8f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cb66da0359c00361dad840499b2b76db06b9e7f10dd4a88de813c5b55cb04f8f","first_computed_at":"2026-07-05T07:44:25.634466Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:44:25.634466Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"pTdCjlYUzqByeAbFZupE8iSX1mqjPS0cKfGxxK1z6XzhFXW8FEZNcYNkq+2RynsH8rZnKbwzM+Ru0S5XfdqFDg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:44:25.634958Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.08005","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:11a4fd10ac8ba90955a808a2033569e57a7230b9b31a5b430139b1f77b749878","sha256:073f632a91b698ab5b789dbfd2a8a0a18e2602476802f422d1e37d3032bdb144"],"state_sha256":"4e9c6e4d7b0110998ff0aa8e5f00254fd183905133f1655de9ca2efe611e60c5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vTC+oO66Ek5m1KkSuj9cf4e6encB0dfuVskxl7SEnpeK9ePdqtj/GXPzpG0EMzejeNWkOgPbo3ueu090GbfuDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T13:50:20.268045Z","bundle_sha256":"1ee1d94c488cdc90766cc3bd00ba2a64400e3be73598d449129cfcc1f28ade5a"}}