{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BCBNDPXUQD7YFGYK3H73W72NC2","short_pith_number":"pith:BCBNDPXU","canonical_record":{"source":{"id":"2408.09385","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-18T07:04:16Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"dfdc7b588fde80da39d08da3e18b3b5e42b853011ae9b896712feff5e93287c9","abstract_canon_sha256":"c11cf7252ce09cc5a96b4aa17cd0ecdf922f8257b0f240e6aa356b03f9a72ff4"},"schema_version":"1.0"},"canonical_sha256":"0882d1bef480ff829b0ad9ffbb7f4d16ad1be03f6818b930efa398ff48ed6206","source":{"kind":"arxiv","id":"2408.09385","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.09385","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"arxiv_version","alias_value":"2408.09385v2","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.09385","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_12","alias_value":"BCBNDPXUQD7Y","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_16","alias_value":"BCBNDPXUQD7YFGYK","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_8","alias_value":"BCBNDPXU","created_at":"2026-07-05T09:28:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BCBNDPXUQD7YFGYK3H73W72NC2","target":"record","payload":{"canonical_record":{"source":{"id":"2408.09385","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-18T07:04:16Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"dfdc7b588fde80da39d08da3e18b3b5e42b853011ae9b896712feff5e93287c9","abstract_canon_sha256":"c11cf7252ce09cc5a96b4aa17cd0ecdf922f8257b0f240e6aa356b03f9a72ff4"},"schema_version":"1.0"},"canonical_sha256":"0882d1bef480ff829b0ad9ffbb7f4d16ad1be03f6818b930efa398ff48ed6206","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:28:13.372499Z","signature_b64":"EFpiZsWkeJ0EJlH9Pf+DGQPgkTFDamIFoVyRoGQKAq/jLrKe0E194zHENWrz9fZX7qyP7CtcNl/f1/lfQY0RDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0882d1bef480ff829b0ad9ffbb7f4d16ad1be03f6818b930efa398ff48ed6206","last_reissued_at":"2026-07-05T09:28:13.372027Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:28:13.372027Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.09385","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:28:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d1gJ/y+WIJ+YXNM1PMGXdphVWiB/UjHsCo+yMOO5eNKJgXY7prwgl8g5COwDi+mIr5MlCgayrpRtQ2J7OhFOAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T08:03:06.868266Z"},"content_sha256":"9fed18b62a9d062103dd200b726db57ffa56eb41232e20e5ceb28c71d323d2ff","schema_version":"1.0","event_id":"sha256:9fed18b62a9d062103dd200b726db57ffa56eb41232e20e5ceb28c71d323d2ff"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BCBNDPXUQD7YFGYK3H73W72NC2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reward Difference Optimization For Sample Reweighting In Offline RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Cam Tu Nguyen, Fei Tan, Rui Zhao, Shiqi Wang, Zhengze Zhang","submitted_at":"2024-08-18T07:04:16Z","abstract_excerpt":"With the rapid advances in Large Language Models (LLMs), aligning LLMs with human preferences become increasingly important. Although Reinforcement Learning with Human Feedback (RLHF) proves effective, it is complicated and highly resource-intensive. As such, offline RLHF has been introduced as an alternative solution, which directly optimizes LLMs with ranking losses on a fixed preference dataset. Current offline RLHF only captures the \"ordinal relationship\" between responses, overlooking the crucial aspect of how much one is preferred over the others. To address this issue, we propose a simp"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.09385","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.09385/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:28:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GD+NVL+buG/GNbqk0ldU4jwwWkEjqlRiy8q61AKjTY0hByp2P4JLcA3HQZKvPJ+fJytdnP+QunkllflUSsQcDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T08:03:06.869226Z"},"content_sha256":"bb5d6a8dd1724ef04888019e56df9bfd99c41452cdcb2dcf757e25d85cf3b8a8","schema_version":"1.0","event_id":"sha256:bb5d6a8dd1724ef04888019e56df9bfd99c41452cdcb2dcf757e25d85cf3b8a8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BCBNDPXUQD7YFGYK3H73W72NC2/bundle.json","state_url":"https://pith.science/pith/BCBNDPXUQD7YFGYK3H73W72NC2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BCBNDPXUQD7YFGYK3H73W72NC2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T08:03:06Z","links":{"resolver":"https://pith.science/pith/BCBNDPXUQD7YFGYK3H73W72NC2","bundle":"https://pith.science/pith/BCBNDPXUQD7YFGYK3H73W72NC2/bundle.json","state":"https://pith.science/pith/BCBNDPXUQD7YFGYK3H73W72NC2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BCBNDPXUQD7YFGYK3H73W72NC2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BCBNDPXUQD7YFGYK3H73W72NC2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c11cf7252ce09cc5a96b4aa17cd0ecdf922f8257b0f240e6aa356b03f9a72ff4","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-18T07:04:16Z","title_canon_sha256":"dfdc7b588fde80da39d08da3e18b3b5e42b853011ae9b896712feff5e93287c9"},"schema_version":"1.0","source":{"id":"2408.09385","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.09385","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"arxiv_version","alias_value":"2408.09385v2","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.09385","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_12","alias_value":"BCBNDPXUQD7Y","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_16","alias_value":"BCBNDPXUQD7YFGYK","created_at":"2026-07-05T09:28:13Z"},{"alias_kind":"pith_short_8","alias_value":"BCBNDPXU","created_at":"2026-07-05T09:28:13Z"}],"graph_snapshots":[{"event_id":"sha256:bb5d6a8dd1724ef04888019e56df9bfd99c41452cdcb2dcf757e25d85cf3b8a8","target":"graph","created_at":"2026-07-05T09:28:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.09385/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"With the rapid advances in Large Language Models (LLMs), aligning LLMs with human preferences become increasingly important. Although Reinforcement Learning with Human Feedback (RLHF) proves effective, it is complicated and highly resource-intensive. As such, offline RLHF has been introduced as an alternative solution, which directly optimizes LLMs with ranking losses on a fixed preference dataset. Current offline RLHF only captures the \"ordinal relationship\" between responses, overlooking the crucial aspect of how much one is preferred over the others. To address this issue, we propose a simp","authors_text":"Cam Tu Nguyen, Fei Tan, Rui Zhao, Shiqi Wang, Zhengze Zhang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-18T07:04:16Z","title":"Reward Difference Optimization For Sample Reweighting In Offline RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.09385","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9fed18b62a9d062103dd200b726db57ffa56eb41232e20e5ceb28c71d323d2ff","target":"record","created_at":"2026-07-05T09:28:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c11cf7252ce09cc5a96b4aa17cd0ecdf922f8257b0f240e6aa356b03f9a72ff4","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-18T07:04:16Z","title_canon_sha256":"dfdc7b588fde80da39d08da3e18b3b5e42b853011ae9b896712feff5e93287c9"},"schema_version":"1.0","source":{"id":"2408.09385","kind":"arxiv","version":2}},"canonical_sha256":"0882d1bef480ff829b0ad9ffbb7f4d16ad1be03f6818b930efa398ff48ed6206","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0882d1bef480ff829b0ad9ffbb7f4d16ad1be03f6818b930efa398ff48ed6206","first_computed_at":"2026-07-05T09:28:13.372027Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:28:13.372027Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"EFpiZsWkeJ0EJlH9Pf+DGQPgkTFDamIFoVyRoGQKAq/jLrKe0E194zHENWrz9fZX7qyP7CtcNl/f1/lfQY0RDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:28:13.372499Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.09385","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9fed18b62a9d062103dd200b726db57ffa56eb41232e20e5ceb28c71d323d2ff","sha256:bb5d6a8dd1724ef04888019e56df9bfd99c41452cdcb2dcf757e25d85cf3b8a8"],"state_sha256":"56487437496a93ab8e0b782a81b27b87592cf2350e7ecc3255b72a2d6cf62384"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e0V/gnOxecAqXn81PQkx7r0dd/e1zq3YppJJ75KF8EWMf2mBWwEhbW18ARR1ofOFY2UaxhtYm6RBLa0c0jJmBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T08:03:06.875833Z","bundle_sha256":"2cc88e4c8fa24cba21109b47056f9b120f91ef17a420265feba68c808fe6ccef"}}