{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:HZ73IXWSSGBO742TUYXZL7HX4V","short_pith_number":"pith:HZ73IXWS","canonical_record":{"source":{"id":"2409.08845","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-13T14:03:49Z","cross_cats_sorted":[],"title_canon_sha256":"2bc33c8104f9dc7e17fccb6b209601af5a0f400b1cca000d8cbdf9ed5773cfe4","abstract_canon_sha256":"56473769e16c6542673345889a4001048b0d7316a4a8bfe533f7b1688d24726c"},"schema_version":"1.0"},"canonical_sha256":"3e7fb45ed29182eff353a62f95fcf7e55b63ac9ab026ce58a4fa48f105597c6f","source":{"kind":"arxiv","id":"2409.08845","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.08845","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"arxiv_version","alias_value":"2409.08845v1","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.08845","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_12","alias_value":"HZ73IXWSSGBO","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_16","alias_value":"HZ73IXWSSGBO742T","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_8","alias_value":"HZ73IXWS","created_at":"2026-07-05T09:06:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:HZ73IXWSSGBO742TUYXZL7HX4V","target":"record","payload":{"canonical_record":{"source":{"id":"2409.08845","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-13T14:03:49Z","cross_cats_sorted":[],"title_canon_sha256":"2bc33c8104f9dc7e17fccb6b209601af5a0f400b1cca000d8cbdf9ed5773cfe4","abstract_canon_sha256":"56473769e16c6542673345889a4001048b0d7316a4a8bfe533f7b1688d24726c"},"schema_version":"1.0"},"canonical_sha256":"3e7fb45ed29182eff353a62f95fcf7e55b63ac9ab026ce58a4fa48f105597c6f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:06:48.780640Z","signature_b64":"WwhmtX109bay1jUNEncNWkRdJOmKgesWc99kolnhwjU2rFh5w32BXa5GCoDhM5Z0sxjAYpoKNpW7wrF+tpLeBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3e7fb45ed29182eff353a62f95fcf7e55b63ac9ab026ce58a4fa48f105597c6f","last_reissued_at":"2026-07-05T09:06:48.780138Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:06:48.780138Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.08845","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:06:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MYqUQFkKmWbGv1g8pZsZSUthDOESBKIbEAWCwOZO/iiYR7CpQqL772vS4vaE5toxTyeeNQNZVRCxBY6M5QyfDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T06:40:36.842210Z"},"content_sha256":"2a5a8a9ead1a390dc008eb78fbf2fdef80ede878cc1ef3e46a18777f57050814","schema_version":"1.0","event_id":"sha256:2a5a8a9ead1a390dc008eb78fbf2fdef80ede878cc1ef3e46a18777f57050814"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:HZ73IXWSSGBO742TUYXZL7HX4V","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"AIPO: Improving Training Objective for Iterative Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Fan Chen, Lexin Tang, Libo Zhang, Longyin Wen, Xinyao Wang, Yaojie Shen, Ying Zhou, Yulei Niu","submitted_at":"2024-09-13T14:03:49Z","abstract_excerpt":"Preference Optimization (PO), is gaining popularity as an alternative choice of Proximal Policy Optimization (PPO) for aligning Large Language Models (LLMs). Recent research on aligning LLMs iteratively with synthetic or partially synthetic data shows promising results in scaling up PO training for both academic settings and proprietary trained models such as Llama3. Despite its success, our study shows that the length exploitation issue present in PO is even more severe in Iterative Preference Optimization (IPO) due to the iterative nature of the process. In this work, we study iterative pref"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.08845","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.08845/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:06:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eC+uEBzQHurm/YQoiRXb1tOke5QlgPgYsFJ2m2D837z6sfCy+VlgYlnMuzyIb/uvUgh7W+DataXkjtMQ/WsBDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T06:40:36.843264Z"},"content_sha256":"5a4691432375552546325e53b583b5f2d4a2d8c7fb6e32c5e192793f38218377","schema_version":"1.0","event_id":"sha256:5a4691432375552546325e53b583b5f2d4a2d8c7fb6e32c5e192793f38218377"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HZ73IXWSSGBO742TUYXZL7HX4V/bundle.json","state_url":"https://pith.science/pith/HZ73IXWSSGBO742TUYXZL7HX4V/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HZ73IXWSSGBO742TUYXZL7HX4V/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T06:40:36Z","links":{"resolver":"https://pith.science/pith/HZ73IXWSSGBO742TUYXZL7HX4V","bundle":"https://pith.science/pith/HZ73IXWSSGBO742TUYXZL7HX4V/bundle.json","state":"https://pith.science/pith/HZ73IXWSSGBO742TUYXZL7HX4V/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HZ73IXWSSGBO742TUYXZL7HX4V/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HZ73IXWSSGBO742TUYXZL7HX4V","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"56473769e16c6542673345889a4001048b0d7316a4a8bfe533f7b1688d24726c","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-13T14:03:49Z","title_canon_sha256":"2bc33c8104f9dc7e17fccb6b209601af5a0f400b1cca000d8cbdf9ed5773cfe4"},"schema_version":"1.0","source":{"id":"2409.08845","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.08845","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"arxiv_version","alias_value":"2409.08845v1","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.08845","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_12","alias_value":"HZ73IXWSSGBO","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_16","alias_value":"HZ73IXWSSGBO742T","created_at":"2026-07-05T09:06:48Z"},{"alias_kind":"pith_short_8","alias_value":"HZ73IXWS","created_at":"2026-07-05T09:06:48Z"}],"graph_snapshots":[{"event_id":"sha256:5a4691432375552546325e53b583b5f2d4a2d8c7fb6e32c5e192793f38218377","target":"graph","created_at":"2026-07-05T09:06:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.08845/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Preference Optimization (PO), is gaining popularity as an alternative choice of Proximal Policy Optimization (PPO) for aligning Large Language Models (LLMs). Recent research on aligning LLMs iteratively with synthetic or partially synthetic data shows promising results in scaling up PO training for both academic settings and proprietary trained models such as Llama3. Despite its success, our study shows that the length exploitation issue present in PO is even more severe in Iterative Preference Optimization (IPO) due to the iterative nature of the process. In this work, we study iterative pref","authors_text":"Fan Chen, Lexin Tang, Libo Zhang, Longyin Wen, Xinyao Wang, Yaojie Shen, Ying Zhou, Yulei Niu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-13T14:03:49Z","title":"AIPO: Improving Training Objective for Iterative Preference Optimization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.08845","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2a5a8a9ead1a390dc008eb78fbf2fdef80ede878cc1ef3e46a18777f57050814","target":"record","created_at":"2026-07-05T09:06:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"56473769e16c6542673345889a4001048b0d7316a4a8bfe533f7b1688d24726c","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-13T14:03:49Z","title_canon_sha256":"2bc33c8104f9dc7e17fccb6b209601af5a0f400b1cca000d8cbdf9ed5773cfe4"},"schema_version":"1.0","source":{"id":"2409.08845","kind":"arxiv","version":1}},"canonical_sha256":"3e7fb45ed29182eff353a62f95fcf7e55b63ac9ab026ce58a4fa48f105597c6f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3e7fb45ed29182eff353a62f95fcf7e55b63ac9ab026ce58a4fa48f105597c6f","first_computed_at":"2026-07-05T09:06:48.780138Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:06:48.780138Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WwhmtX109bay1jUNEncNWkRdJOmKgesWc99kolnhwjU2rFh5w32BXa5GCoDhM5Z0sxjAYpoKNpW7wrF+tpLeBg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:06:48.780640Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.08845","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2a5a8a9ead1a390dc008eb78fbf2fdef80ede878cc1ef3e46a18777f57050814","sha256:5a4691432375552546325e53b583b5f2d4a2d8c7fb6e32c5e192793f38218377"],"state_sha256":"7164824f1a9f781065957541f82911d0a3ca88f84dbe0cd553b1be18a50fe121"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G7JjtRt8Gt5krfv8x3rPsSiXZ2q9p1z16JOUnJF+TIHP1K8CQg4/1ql/3Gmu7ZunPTwWrViWhaq/A/CzY+BzBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T06:40:36.850259Z","bundle_sha256":"691e2240ba901d7b5e726a01a13b05680021a9e1200ac9f52400f4652e6b5ef0"}}