{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:Z4IJ5RAOWRETVMSCWGHULPCQVB","short_pith_number":"pith:Z4IJ5RAO","canonical_record":{"source":{"id":"2406.06874","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-11T01:20:53Z","cross_cats_sorted":["cs.HC","cs.RO"],"title_canon_sha256":"1a8949516bea07d875b8717f91aa6be9def2dfc1b3a034038233e9fea0607687","abstract_canon_sha256":"0c43cc88e264dc942fb1801e7d37f14f33241d5b204d785edab91295bf148226"},"schema_version":"1.0"},"canonical_sha256":"cf109ec40eb4493ab242b18f45bc50a85c2fb57c1bfbba241990cc05e716e260","source":{"kind":"arxiv","id":"2406.06874","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06874","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06874v3","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06874","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_12","alias_value":"Z4IJ5RAOWRET","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_16","alias_value":"Z4IJ5RAOWRETVMSC","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_8","alias_value":"Z4IJ5RAO","created_at":"2026-07-05T09:42:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:Z4IJ5RAOWRETVMSCWGHULPCQVB","target":"record","payload":{"canonical_record":{"source":{"id":"2406.06874","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-11T01:20:53Z","cross_cats_sorted":["cs.HC","cs.RO"],"title_canon_sha256":"1a8949516bea07d875b8717f91aa6be9def2dfc1b3a034038233e9fea0607687","abstract_canon_sha256":"0c43cc88e264dc942fb1801e7d37f14f33241d5b204d785edab91295bf148226"},"schema_version":"1.0"},"canonical_sha256":"cf109ec40eb4493ab242b18f45bc50a85c2fb57c1bfbba241990cc05e716e260","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:42:26.393049Z","signature_b64":"57R2+sq8MCPk8gPM0inO1kbdkuZS6haxuir9lFpKM/YNfk/z4t2rpSEbvhsVPDBHk1w11g4tMy4KWi68TAc2CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cf109ec40eb4493ab242b18f45bc50a85c2fb57c1bfbba241990cc05e716e260","last_reissued_at":"2026-07-05T09:42:26.392525Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:42:26.392525Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.06874","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T1jqDtgEKLCjwzl+mW6gbJ3TeQxx3486Dv4Acs761De2N92T6gfpNrjaTxf1HYYdNeUaAnG4Xtarzlquq6TnDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T08:12:18.701555Z"},"content_sha256":"e07dd9192b8f53cf09ca1420033a84e83c5f1c6cde424d2a0df7f6d7ef393573","schema_version":"1.0","event_id":"sha256:e07dd9192b8f53cf09ca1420033a84e83c5f1c6cde424d2a0df7f6d7ef393573"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:Z4IJ5RAOWRETVMSCWGHULPCQVB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Learning Reward and Policy Jointly from Demonstration and Preference Improves Alignment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.HC","cs.RO"],"primary_cat":"cs.AI","authors_text":"Alfredo Garcia, Chenliang Li, Dongyeop Kang, Jiaxiang Li, Mingyi Hong, Siliang Zeng, Zeyi Liao","submitted_at":"2024-06-11T01:20:53Z","abstract_excerpt":"Aligning human preference and value is an important requirement for building contemporary foundation models and embodied AI. However, popular approaches such as reinforcement learning with human feedback (RLHF) break down the task into successive stages, such as supervised fine-tuning (SFT), reward modeling (RM), and reinforcement learning (RL), each performing one specific learning task. Such a sequential approach results in serious issues such as significant under-utilization of data and distribution mismatch between the learned reward model and generated policy, which eventually lead to poo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06874","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.06874/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0AzM7fY82f5jHme0yjRQEejxgSodImNDSABE6o13NhjclTgAYs1pfhMROtDSG/ygb+fIgo5OznWjftVS1/PfDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T08:12:18.701944Z"},"content_sha256":"087b9cdc2ddb8054a007e5a68b5bb2f818dd5e137403227aacdad831368bbb47","schema_version":"1.0","event_id":"sha256:087b9cdc2ddb8054a007e5a68b5bb2f818dd5e137403227aacdad831368bbb47"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/bundle.json","state_url":"https://pith.science/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T08:12:18Z","links":{"resolver":"https://pith.science/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB","bundle":"https://pith.science/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/bundle.json","state":"https://pith.science/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Z4IJ5RAOWRETVMSCWGHULPCQVB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:Z4IJ5RAOWRETVMSCWGHULPCQVB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0c43cc88e264dc942fb1801e7d37f14f33241d5b204d785edab91295bf148226","cross_cats_sorted":["cs.HC","cs.RO"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-11T01:20:53Z","title_canon_sha256":"1a8949516bea07d875b8717f91aa6be9def2dfc1b3a034038233e9fea0607687"},"schema_version":"1.0","source":{"id":"2406.06874","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06874","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06874v3","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06874","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_12","alias_value":"Z4IJ5RAOWRET","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_16","alias_value":"Z4IJ5RAOWRETVMSC","created_at":"2026-07-05T09:42:26Z"},{"alias_kind":"pith_short_8","alias_value":"Z4IJ5RAO","created_at":"2026-07-05T09:42:26Z"}],"graph_snapshots":[{"event_id":"sha256:087b9cdc2ddb8054a007e5a68b5bb2f818dd5e137403227aacdad831368bbb47","target":"graph","created_at":"2026-07-05T09:42:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.06874/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Aligning human preference and value is an important requirement for building contemporary foundation models and embodied AI. However, popular approaches such as reinforcement learning with human feedback (RLHF) break down the task into successive stages, such as supervised fine-tuning (SFT), reward modeling (RM), and reinforcement learning (RL), each performing one specific learning task. Such a sequential approach results in serious issues such as significant under-utilization of data and distribution mismatch between the learned reward model and generated policy, which eventually lead to poo","authors_text":"Alfredo Garcia, Chenliang Li, Dongyeop Kang, Jiaxiang Li, Mingyi Hong, Siliang Zeng, Zeyi Liao","cross_cats":["cs.HC","cs.RO"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-11T01:20:53Z","title":"Learning Reward and Policy Jointly from Demonstration and Preference Improves Alignment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06874","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e07dd9192b8f53cf09ca1420033a84e83c5f1c6cde424d2a0df7f6d7ef393573","target":"record","created_at":"2026-07-05T09:42:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0c43cc88e264dc942fb1801e7d37f14f33241d5b204d785edab91295bf148226","cross_cats_sorted":["cs.HC","cs.RO"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-11T01:20:53Z","title_canon_sha256":"1a8949516bea07d875b8717f91aa6be9def2dfc1b3a034038233e9fea0607687"},"schema_version":"1.0","source":{"id":"2406.06874","kind":"arxiv","version":3}},"canonical_sha256":"cf109ec40eb4493ab242b18f45bc50a85c2fb57c1bfbba241990cc05e716e260","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cf109ec40eb4493ab242b18f45bc50a85c2fb57c1bfbba241990cc05e716e260","first_computed_at":"2026-07-05T09:42:26.392525Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:42:26.392525Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"57R2+sq8MCPk8gPM0inO1kbdkuZS6haxuir9lFpKM/YNfk/z4t2rpSEbvhsVPDBHk1w11g4tMy4KWi68TAc2CQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:42:26.393049Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.06874","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e07dd9192b8f53cf09ca1420033a84e83c5f1c6cde424d2a0df7f6d7ef393573","sha256:087b9cdc2ddb8054a007e5a68b5bb2f818dd5e137403227aacdad831368bbb47"],"state_sha256":"66482a8c7f6d0f4fc9740b6cbec0f81a16921e0280d786cd3f9f7af7c65d32db"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"a6rRIdSmr7CkvUz9FNMSwSgkIs+3oVn0CR5xFJxNrVIK/eP0XVuYnw975X8QmqGv9Qu/3lODNQHA5SYFJv8KBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T08:12:18.704537Z","bundle_sha256":"e6c2930c87b94133661f0d1603b1b9943fe753bcfb6d3b8b5fd1c83d7d4ba978"}}