{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:E5NPNEQFN6BUMEPHWUPNZIFNS3","short_pith_number":"pith:E5NPNEQF","canonical_record":{"source":{"id":"2410.09362","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T04:17:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"646cc25a14b1c7c17dd20490e40f587696d84421c95054ccf7c7e5eea14ea435","abstract_canon_sha256":"5ef378851da18d7fd9d0434b3e7ac08c103d7543b09fed2c386048d8c57f9ed9"},"schema_version":"1.0"},"canonical_sha256":"275af692056f834611e7b51edca0ad96d2fde0ba10b0c17fe8111d49a4cbe91c","source":{"kind":"arxiv","id":"2410.09362","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.09362","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"arxiv_version","alias_value":"2410.09362v1","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.09362","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_12","alias_value":"E5NPNEQFN6BU","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_16","alias_value":"E5NPNEQFN6BUMEPH","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_8","alias_value":"E5NPNEQF","created_at":"2026-07-05T09:19:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:E5NPNEQFN6BUMEPHWUPNZIFNS3","target":"record","payload":{"canonical_record":{"source":{"id":"2410.09362","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T04:17:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"646cc25a14b1c7c17dd20490e40f587696d84421c95054ccf7c7e5eea14ea435","abstract_canon_sha256":"5ef378851da18d7fd9d0434b3e7ac08c103d7543b09fed2c386048d8c57f9ed9"},"schema_version":"1.0"},"canonical_sha256":"275af692056f834611e7b51edca0ad96d2fde0ba10b0c17fe8111d49a4cbe91c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:19:38.218743Z","signature_b64":"KszVeEs6hj4gWi7F0ZGIBWgtP+pB2XW4hUyQccAx8zBxh1Vnb0LIEU7TbNGm6KPgBoD58RMf+Qe3EoGOG9BsBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"275af692056f834611e7b51edca0ad96d2fde0ba10b0c17fe8111d49a4cbe91c","last_reissued_at":"2026-07-05T09:19:38.218349Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:19:38.218349Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.09362","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AyUnixDHoQFnf/+RwWiLlfcLSwElL3EM3x2zmMwuc8LvonUTGl17CKjHmlby3EHqzR5Zn7AVaj/S5ONQrvc5Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:12:59.493571Z"},"content_sha256":"096fdbf5dcda05018c8a1a770217a48055dd795a27479bad8fdc8cba585e3407","schema_version":"1.0","event_id":"sha256:096fdbf5dcda05018c8a1a770217a48055dd795a27479bad8fdc8cba585e3407"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:E5NPNEQFN6BUMEPHWUPNZIFNS3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SeRA: Self-Reviewing and Alignment of Large Language Models using Implicit Reward Margins","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Aram Galstyan, Bhavana Ganesh, Jongwoo Ko, Sailik Sengupta, Saket Dingliwal, Sravan Bodapati","submitted_at":"2024-10-12T04:17:28Z","abstract_excerpt":"Direct alignment algorithms (DAAs), such as direct preference optimization (DPO), have become popular alternatives for Reinforcement Learning from Human Feedback (RLHF) due to their simplicity, efficiency, and stability. However, the preferences used in DAAs are usually collected before the alignment training begins and remain unchanged (off-policy). This can lead to two problems where the policy model (1) picks up on spurious correlations in the dataset (as opposed to learning the intended alignment expressed in the human preference labels), and (2) overfits to feedback on off-policy trajecto"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.09362","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.09362/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PTCTib3c2RN+KJML11bkUDIFobutipJUj7PBCL9NlGS8ENIwG0I3gqP+MUCdk/rfKP2VUiMJGXO5qKhA68iQBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:12:59.493955Z"},"content_sha256":"4c4921402f4578d1957a74fa9f4ea9765422e50073455acccbd430f05c0b9633","schema_version":"1.0","event_id":"sha256:4c4921402f4578d1957a74fa9f4ea9765422e50073455acccbd430f05c0b9633"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/bundle.json","state_url":"https://pith.science/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T04:12:59Z","links":{"resolver":"https://pith.science/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3","bundle":"https://pith.science/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/bundle.json","state":"https://pith.science/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/E5NPNEQFN6BUMEPHWUPNZIFNS3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:E5NPNEQFN6BUMEPHWUPNZIFNS3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5ef378851da18d7fd9d0434b3e7ac08c103d7543b09fed2c386048d8c57f9ed9","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T04:17:28Z","title_canon_sha256":"646cc25a14b1c7c17dd20490e40f587696d84421c95054ccf7c7e5eea14ea435"},"schema_version":"1.0","source":{"id":"2410.09362","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.09362","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"arxiv_version","alias_value":"2410.09362v1","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.09362","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_12","alias_value":"E5NPNEQFN6BU","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_16","alias_value":"E5NPNEQFN6BUMEPH","created_at":"2026-07-05T09:19:38Z"},{"alias_kind":"pith_short_8","alias_value":"E5NPNEQF","created_at":"2026-07-05T09:19:38Z"}],"graph_snapshots":[{"event_id":"sha256:4c4921402f4578d1957a74fa9f4ea9765422e50073455acccbd430f05c0b9633","target":"graph","created_at":"2026-07-05T09:19:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.09362/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Direct alignment algorithms (DAAs), such as direct preference optimization (DPO), have become popular alternatives for Reinforcement Learning from Human Feedback (RLHF) due to their simplicity, efficiency, and stability. However, the preferences used in DAAs are usually collected before the alignment training begins and remain unchanged (off-policy). This can lead to two problems where the policy model (1) picks up on spurious correlations in the dataset (as opposed to learning the intended alignment expressed in the human preference labels), and (2) overfits to feedback on off-policy trajecto","authors_text":"Aram Galstyan, Bhavana Ganesh, Jongwoo Ko, Sailik Sengupta, Saket Dingliwal, Sravan Bodapati","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T04:17:28Z","title":"SeRA: Self-Reviewing and Alignment of Large Language Models using Implicit Reward Margins"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.09362","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:096fdbf5dcda05018c8a1a770217a48055dd795a27479bad8fdc8cba585e3407","target":"record","created_at":"2026-07-05T09:19:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5ef378851da18d7fd9d0434b3e7ac08c103d7543b09fed2c386048d8c57f9ed9","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-12T04:17:28Z","title_canon_sha256":"646cc25a14b1c7c17dd20490e40f587696d84421c95054ccf7c7e5eea14ea435"},"schema_version":"1.0","source":{"id":"2410.09362","kind":"arxiv","version":1}},"canonical_sha256":"275af692056f834611e7b51edca0ad96d2fde0ba10b0c17fe8111d49a4cbe91c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"275af692056f834611e7b51edca0ad96d2fde0ba10b0c17fe8111d49a4cbe91c","first_computed_at":"2026-07-05T09:19:38.218349Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:19:38.218349Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KszVeEs6hj4gWi7F0ZGIBWgtP+pB2XW4hUyQccAx8zBxh1Vnb0LIEU7TbNGm6KPgBoD58RMf+Qe3EoGOG9BsBg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:19:38.218743Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.09362","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:096fdbf5dcda05018c8a1a770217a48055dd795a27479bad8fdc8cba585e3407","sha256:4c4921402f4578d1957a74fa9f4ea9765422e50073455acccbd430f05c0b9633"],"state_sha256":"075b9999184c0e141710942673a49a0a972a86540c84520155a38a9ed5c35beb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B82NjBr4Cs2gJrj3CeP9yWQT61lpH+g0whPqcakIJX+ztSEd+ea8tgWJZjoWKldIUuOHPKfyudtZotPZc3gSBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T04:12:59.495997Z","bundle_sha256":"e4e535175772257d3d82e44cb35ee56ce597e78a055cb98a5506a7ac83befdcf"}}