{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:TCLDYCN5F7AKSRR3ZK7UKQ4OEU","short_pith_number":"pith:TCLDYCN5","canonical_record":{"source":{"id":"2402.10184","kind":"arxiv","version":7},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-15T18:39:24Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DM"],"title_canon_sha256":"19fb4c4d047479cd98f386ff62c4b1004714974ccc8e6cd7df0ed6934b872ede","abstract_canon_sha256":"9378df911dbc3a81679bb8ae5f0634919dcc2348af5d20959bef2659853d45b7"},"schema_version":"1.0"},"canonical_sha256":"98963c09bd2fc0a9463bcabf45438e252a31ffc5a38ebe252d7eac4071604c8c","source":{"kind":"arxiv","id":"2402.10184","version":7},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10184","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10184v7","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10184","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_12","alias_value":"TCLDYCN5F7AK","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_16","alias_value":"TCLDYCN5F7AKSRR3","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_8","alias_value":"TCLDYCN5","created_at":"2026-07-05T11:10:45Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:TCLDYCN5F7AKSRR3ZK7UKQ4OEU","target":"record","payload":{"canonical_record":{"source":{"id":"2402.10184","kind":"arxiv","version":7},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-15T18:39:24Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DM"],"title_canon_sha256":"19fb4c4d047479cd98f386ff62c4b1004714974ccc8e6cd7df0ed6934b872ede","abstract_canon_sha256":"9378df911dbc3a81679bb8ae5f0634919dcc2348af5d20959bef2659853d45b7"},"schema_version":"1.0"},"canonical_sha256":"98963c09bd2fc0a9463bcabf45438e252a31ffc5a38ebe252d7eac4071604c8c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:10:45.110841Z","signature_b64":"B1UwBF189DczJfbRzStB+wNKTHai++eCLyTsF0BV1Gp//L7VbiGTaA3cJ8Ta6QVjmie4OUu1bEF0HnhzBFKXBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"98963c09bd2fc0a9463bcabf45438e252a31ffc5a38ebe252d7eac4071604c8c","last_reissued_at":"2026-07-05T11:10:45.110330Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:10:45.110330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.10184","source_version":7,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/RFkqucEYHUBKsyc/dBU/5iKKiZ41mdKCuJEK1d17yM4r91+T/eAVTgxTs6wBNwHCYWgGGMg5pzBuxAPLbldBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T21:03:15.055183Z"},"content_sha256":"257513f77456b4df9d654408fa509c24ca14a05495a9b702a6aec63632076d2e","schema_version":"1.0","event_id":"sha256:257513f77456b4df9d654408fa509c24ca14a05495a9b702a6aec63632076d2e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:TCLDYCN5F7AKSRR3ZK7UKQ4OEU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reward Generalization in RLHF: A Topological Perspective","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.DM"],"primary_cat":"cs.LG","authors_text":"Dong Yan, Fanzhi Zeng, Jiaming Ji, Jiayi Zhou, Josef Dai, Kaile Wang, Tianyi Qiu, Xuehai Pan, Yang Han, Yaodong Yang","submitted_at":"2024-02-15T18:39:24Z","abstract_excerpt":"Existing alignment methods share a common topology of information flow, where reward information is collected from humans, modeled with preference learning, and used to tune language models. However, this shared topology has not been systematically characterized, nor have its alternatives been thoroughly explored, leaving the problems of low data efficiency and unreliable generalization unaddressed. As a solution, we introduce a theory of reward generalization in reinforcement learning from human feedback (RLHF), focusing on the topology of information flow at both macro and micro levels. At t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10184","kind":"arxiv","version":7},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.10184/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ACCLPwu6AfSTWga7DEpuzHGYrwqezyMsFvqcczK71lKwqcJK7NJt+gOKkFUJDnQUnPV8czdS65IfPo84+e7RAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T21:03:15.055530Z"},"content_sha256":"5a26cbb4a73af39b472be0f3af671e41d75d600f83dafde4fb894657f9642251","schema_version":"1.0","event_id":"sha256:5a26cbb4a73af39b472be0f3af671e41d75d600f83dafde4fb894657f9642251"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/bundle.json","state_url":"https://pith.science/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T21:03:15Z","links":{"resolver":"https://pith.science/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU","bundle":"https://pith.science/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/bundle.json","state":"https://pith.science/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TCLDYCN5F7AKSRR3ZK7UKQ4OEU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TCLDYCN5F7AKSRR3ZK7UKQ4OEU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9378df911dbc3a81679bb8ae5f0634919dcc2348af5d20959bef2659853d45b7","cross_cats_sorted":["cs.AI","cs.CL","cs.DM"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-15T18:39:24Z","title_canon_sha256":"19fb4c4d047479cd98f386ff62c4b1004714974ccc8e6cd7df0ed6934b872ede"},"schema_version":"1.0","source":{"id":"2402.10184","kind":"arxiv","version":7}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10184","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10184v7","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10184","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_12","alias_value":"TCLDYCN5F7AK","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_16","alias_value":"TCLDYCN5F7AKSRR3","created_at":"2026-07-05T11:10:45Z"},{"alias_kind":"pith_short_8","alias_value":"TCLDYCN5","created_at":"2026-07-05T11:10:45Z"}],"graph_snapshots":[{"event_id":"sha256:5a26cbb4a73af39b472be0f3af671e41d75d600f83dafde4fb894657f9642251","target":"graph","created_at":"2026-07-05T11:10:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.10184/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Existing alignment methods share a common topology of information flow, where reward information is collected from humans, modeled with preference learning, and used to tune language models. However, this shared topology has not been systematically characterized, nor have its alternatives been thoroughly explored, leaving the problems of low data efficiency and unreliable generalization unaddressed. As a solution, we introduce a theory of reward generalization in reinforcement learning from human feedback (RLHF), focusing on the topology of information flow at both macro and micro levels. At t","authors_text":"Dong Yan, Fanzhi Zeng, Jiaming Ji, Jiayi Zhou, Josef Dai, Kaile Wang, Tianyi Qiu, Xuehai Pan, Yang Han, Yaodong Yang","cross_cats":["cs.AI","cs.CL","cs.DM"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-15T18:39:24Z","title":"Reward Generalization in RLHF: A Topological Perspective"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10184","kind":"arxiv","version":7},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:257513f77456b4df9d654408fa509c24ca14a05495a9b702a6aec63632076d2e","target":"record","created_at":"2026-07-05T11:10:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9378df911dbc3a81679bb8ae5f0634919dcc2348af5d20959bef2659853d45b7","cross_cats_sorted":["cs.AI","cs.CL","cs.DM"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-15T18:39:24Z","title_canon_sha256":"19fb4c4d047479cd98f386ff62c4b1004714974ccc8e6cd7df0ed6934b872ede"},"schema_version":"1.0","source":{"id":"2402.10184","kind":"arxiv","version":7}},"canonical_sha256":"98963c09bd2fc0a9463bcabf45438e252a31ffc5a38ebe252d7eac4071604c8c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"98963c09bd2fc0a9463bcabf45438e252a31ffc5a38ebe252d7eac4071604c8c","first_computed_at":"2026-07-05T11:10:45.110330Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:10:45.110330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"B1UwBF189DczJfbRzStB+wNKTHai++eCLyTsF0BV1Gp//L7VbiGTaA3cJ8Ta6QVjmie4OUu1bEF0HnhzBFKXBw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:10:45.110841Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.10184","source_kind":"arxiv","source_version":7}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:257513f77456b4df9d654408fa509c24ca14a05495a9b702a6aec63632076d2e","sha256:5a26cbb4a73af39b472be0f3af671e41d75d600f83dafde4fb894657f9642251"],"state_sha256":"6f642f378d5f6218d69a4344794db5a1b9399683e0e6f9933d0a85623769fbb6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ilzv6RqrNuQ64Fs+Vd0iACB7YHOCMrwEXiVEL0RshUFm+R5J0ECqDrNmxeYvfVSodotXWfMhbpa3cwRRa7CMBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T21:03:15.058798Z","bundle_sha256":"45ad912e2f3217c96d88078f9a4d777002f24ef37c6222bab5a1f1e5e6c8af57"}}