{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:73Z72HUCCAEL7IRMEG5RUQGG7H","short_pith_number":"pith:73Z72HUC","canonical_record":{"source":{"id":"2406.09760","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T06:57:18Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"07600ad41ac3e1dad180d1621b2d97ed50d08eabce913b0d958359dc363c44ae","abstract_canon_sha256":"7128e2f664a28b90e2f44455e5072bdf89b92ca10fdb86295332cd656b0c2bb5"},"schema_version":"1.0"},"canonical_sha256":"fef3fd1e821008bfa22c21bb1a40c6f9c7021602c1bac12683601e567651029d","source":{"kind":"arxiv","id":"2406.09760","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09760","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09760v2","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09760","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_12","alias_value":"73Z72HUCCAEL","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_16","alias_value":"73Z72HUCCAEL7IRM","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_8","alias_value":"73Z72HUC","created_at":"2026-07-05T10:26:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:73Z72HUCCAEL7IRMEG5RUQGG7H","target":"record","payload":{"canonical_record":{"source":{"id":"2406.09760","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T06:57:18Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"07600ad41ac3e1dad180d1621b2d97ed50d08eabce913b0d958359dc363c44ae","abstract_canon_sha256":"7128e2f664a28b90e2f44455e5072bdf89b92ca10fdb86295332cd656b0c2bb5"},"schema_version":"1.0"},"canonical_sha256":"fef3fd1e821008bfa22c21bb1a40c6f9c7021602c1bac12683601e567651029d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:26:04.502948Z","signature_b64":"hzdaZY9vguOTKKk4T2BWrl4dIUThYzHcyln71l+tFWO72Oo9wuxvBQp5MlL+C1dNcGgBP0zh8+cE8KX01n2WDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fef3fd1e821008bfa22c21bb1a40c6f9c7021602c1bac12683601e567651029d","last_reissued_at":"2026-07-05T10:26:04.502237Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:26:04.502237Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.09760","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:26:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/QHIQ4ZZKrMSwnBAWMo3CMH6nHWTrfuJAQzGrHbzWaFmhbCnd0ICX2pILPdKadr9qub0zgRFdQuZ2qF2gpFnBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:34:04.181073Z"},"content_sha256":"a485a9ab23d6c43b5173d384e3be05655806def51a9165a30d515ec07cd45dad","schema_version":"1.0","event_id":"sha256:a485a9ab23d6c43b5173d384e3be05655806def51a9165a30d515ec07cd45dad"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:73Z72HUCCAEL7IRMEG5RUQGG7H","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Bootstrapping Language Models with DPO Implicit Rewards","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Arunesh Sinha, Changyu Chen, Chao Du, Min Lin, Pradeep Varakantham, Qian Liu, Tianyu Pang, Zichen Liu","submitted_at":"2024-06-14T06:57:18Z","abstract_excerpt":"Human alignment in large language models (LLMs) is an active area of research. A recent groundbreaking work, direct preference optimization (DPO), has greatly simplified the process from past work in reinforcement learning from human feedback (RLHF) by bypassing the reward learning stage in RLHF. DPO, after training, provides an implicit reward model. In this work, we make a novel observation that this implicit reward model can by itself be used in a bootstrapping fashion to further align the LLM. Our approach is to use the rewards from a current LLM to construct a preference dataset, which is"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09760","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.09760/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:26:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Yj1jKLull2fPyBGsXYQ2W0YOx09DDgVeOp2P8PM/eSMEaqimdNJ6zlChGO8LNNsM6e67OJvCBddju4PRURLlAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:34:04.181622Z"},"content_sha256":"d3784541d1ae2c3ed764a748c3c4f3439997c38bd11ff8256a010ea385f2f5f1","schema_version":"1.0","event_id":"sha256:d3784541d1ae2c3ed764a748c3c4f3439997c38bd11ff8256a010ea385f2f5f1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/bundle.json","state_url":"https://pith.science/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T09:34:04Z","links":{"resolver":"https://pith.science/pith/73Z72HUCCAEL7IRMEG5RUQGG7H","bundle":"https://pith.science/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/bundle.json","state":"https://pith.science/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/state.json","well_known_bundle":"https://pith.science/.well-known/pith/73Z72HUCCAEL7IRMEG5RUQGG7H/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:73Z72HUCCAEL7IRMEG5RUQGG7H","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7128e2f664a28b90e2f44455e5072bdf89b92ca10fdb86295332cd656b0c2bb5","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T06:57:18Z","title_canon_sha256":"07600ad41ac3e1dad180d1621b2d97ed50d08eabce913b0d958359dc363c44ae"},"schema_version":"1.0","source":{"id":"2406.09760","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09760","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09760v2","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09760","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_12","alias_value":"73Z72HUCCAEL","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_16","alias_value":"73Z72HUCCAEL7IRM","created_at":"2026-07-05T10:26:04Z"},{"alias_kind":"pith_short_8","alias_value":"73Z72HUC","created_at":"2026-07-05T10:26:04Z"}],"graph_snapshots":[{"event_id":"sha256:d3784541d1ae2c3ed764a748c3c4f3439997c38bd11ff8256a010ea385f2f5f1","target":"graph","created_at":"2026-07-05T10:26:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.09760/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Human alignment in large language models (LLMs) is an active area of research. A recent groundbreaking work, direct preference optimization (DPO), has greatly simplified the process from past work in reinforcement learning from human feedback (RLHF) by bypassing the reward learning stage in RLHF. DPO, after training, provides an implicit reward model. In this work, we make a novel observation that this implicit reward model can by itself be used in a bootstrapping fashion to further align the LLM. Our approach is to use the rewards from a current LLM to construct a preference dataset, which is","authors_text":"Arunesh Sinha, Changyu Chen, Chao Du, Min Lin, Pradeep Varakantham, Qian Liu, Tianyu Pang, Zichen Liu","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T06:57:18Z","title":"Bootstrapping Language Models with DPO Implicit Rewards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09760","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a485a9ab23d6c43b5173d384e3be05655806def51a9165a30d515ec07cd45dad","target":"record","created_at":"2026-07-05T10:26:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7128e2f664a28b90e2f44455e5072bdf89b92ca10fdb86295332cd656b0c2bb5","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T06:57:18Z","title_canon_sha256":"07600ad41ac3e1dad180d1621b2d97ed50d08eabce913b0d958359dc363c44ae"},"schema_version":"1.0","source":{"id":"2406.09760","kind":"arxiv","version":2}},"canonical_sha256":"fef3fd1e821008bfa22c21bb1a40c6f9c7021602c1bac12683601e567651029d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fef3fd1e821008bfa22c21bb1a40c6f9c7021602c1bac12683601e567651029d","first_computed_at":"2026-07-05T10:26:04.502237Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:26:04.502237Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"hzdaZY9vguOTKKk4T2BWrl4dIUThYzHcyln71l+tFWO72Oo9wuxvBQp5MlL+C1dNcGgBP0zh8+cE8KX01n2WDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:26:04.502948Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.09760","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a485a9ab23d6c43b5173d384e3be05655806def51a9165a30d515ec07cd45dad","sha256:d3784541d1ae2c3ed764a748c3c4f3439997c38bd11ff8256a010ea385f2f5f1"],"state_sha256":"8d8002b08d64787de9259d682e8b4c130aa5632b664e223b7204bd65656d59f1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3gJneIL/LMgkmmGrln+AUYXq8OcvX+It+r1KN26k3ZmWD8sh5yBCzzTD5l3sCb78A014dP6fcqlWr8NqmT5iDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T09:34:04.187067Z","bundle_sha256":"a12ec14b73991665670b9312d75d379d9796da6288716565cc1b6eb5d14b8c12"}}