{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:R5EIXFYIMHLSVMZBHSMM2GDRTZ","short_pith_number":"pith:R5EIXFYI","canonical_record":{"source":{"id":"2408.12130","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-08-22T04:54:25Z","cross_cats_sorted":[],"title_canon_sha256":"c890c86d042282c492478e0444a3eebb178a740248ca077514bce73abecacc65","abstract_canon_sha256":"d5286956e8a0d270c968ad9c24699de89c122b30d3701572eb18d55a054ac382"},"schema_version":"1.0"},"canonical_sha256":"8f488b970861d72ab3213c98cd18719e4dd70c46e395fdd9f1700a42c9443211","source":{"kind":"arxiv","id":"2408.12130","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.12130","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"arxiv_version","alias_value":"2408.12130v3","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.12130","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_12","alias_value":"R5EIXFYIMHLS","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_16","alias_value":"R5EIXFYIMHLSVMZB","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_8","alias_value":"R5EIXFYI","created_at":"2026-07-05T11:02:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:R5EIXFYIMHLSVMZBHSMM2GDRTZ","target":"record","payload":{"canonical_record":{"source":{"id":"2408.12130","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-08-22T04:54:25Z","cross_cats_sorted":[],"title_canon_sha256":"c890c86d042282c492478e0444a3eebb178a740248ca077514bce73abecacc65","abstract_canon_sha256":"d5286956e8a0d270c968ad9c24699de89c122b30d3701572eb18d55a054ac382"},"schema_version":"1.0"},"canonical_sha256":"8f488b970861d72ab3213c98cd18719e4dd70c46e395fdd9f1700a42c9443211","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:02:12.264162Z","signature_b64":"g569Cc8+e7bxSt9SL6Wxf4TPtRQaaDqYhusOJcNCZVCX91F/+9k7B4rO7aFRmnsO+L0QJ+KPpLc3hjsluw4bAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8f488b970861d72ab3213c98cd18719e4dd70c46e395fdd9f1700a42c9443211","last_reissued_at":"2026-07-05T11:02:12.263676Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:02:12.263676Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.12130","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:02:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"x7VN9igWY+2XwlMPPSZz8wdJYxjYF8WU0Ca7xeDFN1yBAXFuPAw4vOr83w8AzD3AaPGMlTbaS1x4fixkkNktBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:03:25.697027Z"},"content_sha256":"3f8de5e803ba8b824ff8245333a69bf9f169ccc82225e685b6e32851922ef091","schema_version":"1.0","event_id":"sha256:3f8de5e803ba8b824ff8245333a69bf9f169ccc82225e685b6e32851922ef091"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:R5EIXFYIMHLSVMZBHSMM2GDRTZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"S-EPOA: Overcoming the Indistinguishability of Segments with Skill-Driven Preference-Based Reinforcement Learning","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Bo Xu, Ni Mu, Qing-Shan Jia, Yao Luan, Yiqin Yang","submitted_at":"2024-08-22T04:54:25Z","abstract_excerpt":"Preference-based reinforcement learning (PbRL) stands out by utilizing human preferences as a direct reward signal, eliminating the need for intricate reward engineering. However, despite its potential, traditional PbRL methods are often constrained by the indistinguishability of segments, which impedes the learning process. In this paper, we introduce Skill-Enhanced Preference Optimization Algorithm (S-EPOA), which addresses the segment indistinguishability issue by integrating skill mechanisms into the preference learning framework. Specifically, we first conduct the unsupervised pretraining"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.12130","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.12130/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:02:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xqGnWg4Z24l0UgRxDxtXowuQjW+R/Sp+G45gV7DUkb4P7YtqUXCzOjwPDyophv3RmzBS+EdTwCJNqeV8FkgTDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:03:25.697538Z"},"content_sha256":"047077a6235b2cf9c55ae347583e72c0523ba73c648e80a664023585e7b769af","schema_version":"1.0","event_id":"sha256:047077a6235b2cf9c55ae347583e72c0523ba73c648e80a664023585e7b769af"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/bundle.json","state_url":"https://pith.science/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T09:03:25Z","links":{"resolver":"https://pith.science/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ","bundle":"https://pith.science/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/bundle.json","state":"https://pith.science/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/R5EIXFYIMHLSVMZBHSMM2GDRTZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:R5EIXFYIMHLSVMZBHSMM2GDRTZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d5286956e8a0d270c968ad9c24699de89c122b30d3701572eb18d55a054ac382","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-08-22T04:54:25Z","title_canon_sha256":"c890c86d042282c492478e0444a3eebb178a740248ca077514bce73abecacc65"},"schema_version":"1.0","source":{"id":"2408.12130","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.12130","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"arxiv_version","alias_value":"2408.12130v3","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.12130","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_12","alias_value":"R5EIXFYIMHLS","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_16","alias_value":"R5EIXFYIMHLSVMZB","created_at":"2026-07-05T11:02:12Z"},{"alias_kind":"pith_short_8","alias_value":"R5EIXFYI","created_at":"2026-07-05T11:02:12Z"}],"graph_snapshots":[{"event_id":"sha256:047077a6235b2cf9c55ae347583e72c0523ba73c648e80a664023585e7b769af","target":"graph","created_at":"2026-07-05T11:02:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.12130/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Preference-based reinforcement learning (PbRL) stands out by utilizing human preferences as a direct reward signal, eliminating the need for intricate reward engineering. However, despite its potential, traditional PbRL methods are often constrained by the indistinguishability of segments, which impedes the learning process. In this paper, we introduce Skill-Enhanced Preference Optimization Algorithm (S-EPOA), which addresses the segment indistinguishability issue by integrating skill mechanisms into the preference learning framework. Specifically, we first conduct the unsupervised pretraining","authors_text":"Bo Xu, Ni Mu, Qing-Shan Jia, Yao Luan, Yiqin Yang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-08-22T04:54:25Z","title":"S-EPOA: Overcoming the Indistinguishability of Segments with Skill-Driven Preference-Based Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.12130","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3f8de5e803ba8b824ff8245333a69bf9f169ccc82225e685b6e32851922ef091","target":"record","created_at":"2026-07-05T11:02:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d5286956e8a0d270c968ad9c24699de89c122b30d3701572eb18d55a054ac382","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-08-22T04:54:25Z","title_canon_sha256":"c890c86d042282c492478e0444a3eebb178a740248ca077514bce73abecacc65"},"schema_version":"1.0","source":{"id":"2408.12130","kind":"arxiv","version":3}},"canonical_sha256":"8f488b970861d72ab3213c98cd18719e4dd70c46e395fdd9f1700a42c9443211","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8f488b970861d72ab3213c98cd18719e4dd70c46e395fdd9f1700a42c9443211","first_computed_at":"2026-07-05T11:02:12.263676Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:02:12.263676Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"g569Cc8+e7bxSt9SL6Wxf4TPtRQaaDqYhusOJcNCZVCX91F/+9k7B4rO7aFRmnsO+L0QJ+KPpLc3hjsluw4bAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:02:12.264162Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.12130","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3f8de5e803ba8b824ff8245333a69bf9f169ccc82225e685b6e32851922ef091","sha256:047077a6235b2cf9c55ae347583e72c0523ba73c648e80a664023585e7b769af"],"state_sha256":"123cecd48a5dc6a4a8c4294a85715a8cda59af49c37c54990b3a318156adc1ed"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"27so4CgKWWFmuAexWb0pK7V9Vxrl6bOkOHLrcbbM1VlHK/f+1CHcSSbsdypYvLwMzSHljyPtiyj8zjDo/FleDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T09:03:25.702486Z","bundle_sha256":"527bb54a974d08e262588a771c6a11b8135a168ba31c345f714bd20f6b5ea4f7"}}