{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:LNHEFSO5GONKQVHGZMQML5UO3C","short_pith_number":"pith:LNHEFSO5","canonical_record":{"source":{"id":"2505.14946","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-20T22:14:44Z","cross_cats_sorted":[],"title_canon_sha256":"51cb1a00e13fa2b9eba16333b8149375303dac65caf39355c554077606aec78d","abstract_canon_sha256":"3664ac622f819f82961b3a30435ba3f53794a30a23ad104ab18ee2ae6d2f2393"},"schema_version":"1.0"},"canonical_sha256":"5b4e42c9dd339aa854e6cb20c5f68ed8973e7ec276eb3f2aaffcd2367369e592","source":{"kind":"arxiv","id":"2505.14946","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.14946","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"arxiv_version","alias_value":"2505.14946v1","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.14946","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_12","alias_value":"LNHEFSO5GONK","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_16","alias_value":"LNHEFSO5GONKQVHG","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_8","alias_value":"LNHEFSO5","created_at":"2026-07-05T11:06:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:LNHEFSO5GONKQVHGZMQML5UO3C","target":"record","payload":{"canonical_record":{"source":{"id":"2505.14946","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-20T22:14:44Z","cross_cats_sorted":[],"title_canon_sha256":"51cb1a00e13fa2b9eba16333b8149375303dac65caf39355c554077606aec78d","abstract_canon_sha256":"3664ac622f819f82961b3a30435ba3f53794a30a23ad104ab18ee2ae6d2f2393"},"schema_version":"1.0"},"canonical_sha256":"5b4e42c9dd339aa854e6cb20c5f68ed8973e7ec276eb3f2aaffcd2367369e592","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:06:38.020860Z","signature_b64":"rtlHQ8pKw5+8s9XH8qRKgkKO+CKwGAxOwvtZTca1sjPDyTg0xIc423KjA2UWVIbZDp5wGB2f72bq0pyYpJLdCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5b4e42c9dd339aa854e6cb20c5f68ed8973e7ec276eb3f2aaffcd2367369e592","last_reissued_at":"2026-07-05T11:06:38.020343Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:06:38.020343Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.14946","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eEHDuktmPVn2FHBoWTtvlCVCrT/h9Lq6pdb61fI3XnyN7sWMezhl74z/PrWmY3J2FDRIpqJIYV1A8lIo+PrbDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:45:17.541551Z"},"content_sha256":"9335730890363996363c1c635a359d7676d8c17efdb691dd2da18d9ea3523af6","schema_version":"1.0","event_id":"sha256:9335730890363996363c1c635a359d7676d8c17efdb691dd2da18d9ea3523af6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:LNHEFSO5GONKQVHGZMQML5UO3C","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reinforcement Learning from User Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Arya Talebzadeh, Eric Han, Eryk Helenowski, Han Fang, Jun Chen, Kaiyan Peng, Karthik Abinav Sankararaman, Mrinal Kumar, Sinong Wang, Tengyu Xu, Xiaoliang Peng","submitted_at":"2025-05-20T22:14:44Z","abstract_excerpt":"As large language models (LLMs) are increasingly deployed in diverse user facing applications, aligning them with real user preferences becomes essential. Existing methods like Reinforcement Learning from Human Feedback (RLHF) rely on expert annotators trained on manually defined guidelines, whose judgments may not reflect the priorities of everyday users. We introduce Reinforcement Learning from User Feedback (RLUF), a framework for aligning LLMs directly to implicit signals from users in production. RLUF addresses key challenges of user feedback: user feedback is often binary (e.g., emoji re"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.14946","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.14946/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eOKYIE4iXRbbpi6YxS2RYgJOutpXYZXNG5f278cSb/zQHX7dYoUEZGRo2bEKIoHPKYjX7QvIBBEzYOAE4YubAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:45:17.541944Z"},"content_sha256":"252eef1c38731a89e971bf4bea0724364a95fb0347717d9f4988d6d966976351","schema_version":"1.0","event_id":"sha256:252eef1c38731a89e971bf4bea0724364a95fb0347717d9f4988d6d966976351"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LNHEFSO5GONKQVHGZMQML5UO3C/bundle.json","state_url":"https://pith.science/pith/LNHEFSO5GONKQVHGZMQML5UO3C/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LNHEFSO5GONKQVHGZMQML5UO3C/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T15:45:17Z","links":{"resolver":"https://pith.science/pith/LNHEFSO5GONKQVHGZMQML5UO3C","bundle":"https://pith.science/pith/LNHEFSO5GONKQVHGZMQML5UO3C/bundle.json","state":"https://pith.science/pith/LNHEFSO5GONKQVHGZMQML5UO3C/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LNHEFSO5GONKQVHGZMQML5UO3C/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:LNHEFSO5GONKQVHGZMQML5UO3C","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3664ac622f819f82961b3a30435ba3f53794a30a23ad104ab18ee2ae6d2f2393","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-20T22:14:44Z","title_canon_sha256":"51cb1a00e13fa2b9eba16333b8149375303dac65caf39355c554077606aec78d"},"schema_version":"1.0","source":{"id":"2505.14946","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.14946","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"arxiv_version","alias_value":"2505.14946v1","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.14946","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_12","alias_value":"LNHEFSO5GONK","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_16","alias_value":"LNHEFSO5GONKQVHG","created_at":"2026-07-05T11:06:38Z"},{"alias_kind":"pith_short_8","alias_value":"LNHEFSO5","created_at":"2026-07-05T11:06:38Z"}],"graph_snapshots":[{"event_id":"sha256:252eef1c38731a89e971bf4bea0724364a95fb0347717d9f4988d6d966976351","target":"graph","created_at":"2026-07-05T11:06:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.14946/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As large language models (LLMs) are increasingly deployed in diverse user facing applications, aligning them with real user preferences becomes essential. Existing methods like Reinforcement Learning from Human Feedback (RLHF) rely on expert annotators trained on manually defined guidelines, whose judgments may not reflect the priorities of everyday users. We introduce Reinforcement Learning from User Feedback (RLUF), a framework for aligning LLMs directly to implicit signals from users in production. RLUF addresses key challenges of user feedback: user feedback is often binary (e.g., emoji re","authors_text":"Arya Talebzadeh, Eric Han, Eryk Helenowski, Han Fang, Jun Chen, Kaiyan Peng, Karthik Abinav Sankararaman, Mrinal Kumar, Sinong Wang, Tengyu Xu, Xiaoliang Peng","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-20T22:14:44Z","title":"Reinforcement Learning from User Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.14946","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9335730890363996363c1c635a359d7676d8c17efdb691dd2da18d9ea3523af6","target":"record","created_at":"2026-07-05T11:06:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3664ac622f819f82961b3a30435ba3f53794a30a23ad104ab18ee2ae6d2f2393","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-20T22:14:44Z","title_canon_sha256":"51cb1a00e13fa2b9eba16333b8149375303dac65caf39355c554077606aec78d"},"schema_version":"1.0","source":{"id":"2505.14946","kind":"arxiv","version":1}},"canonical_sha256":"5b4e42c9dd339aa854e6cb20c5f68ed8973e7ec276eb3f2aaffcd2367369e592","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5b4e42c9dd339aa854e6cb20c5f68ed8973e7ec276eb3f2aaffcd2367369e592","first_computed_at":"2026-07-05T11:06:38.020343Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:06:38.020343Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"rtlHQ8pKw5+8s9XH8qRKgkKO+CKwGAxOwvtZTca1sjPDyTg0xIc423KjA2UWVIbZDp5wGB2f72bq0pyYpJLdCw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:06:38.020860Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.14946","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9335730890363996363c1c635a359d7676d8c17efdb691dd2da18d9ea3523af6","sha256:252eef1c38731a89e971bf4bea0724364a95fb0347717d9f4988d6d966976351"],"state_sha256":"6e72089f58e2764b5f2690ec68ffe2efd7892f3ce3b65620b72a94c260aed439"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CAqPJu60AZ1qsC1pl4ZHej0/nL5pc9pwOFLpA7fIco/2Afj5YPJBpDXdsNBVn4a+Uo2rELcAKb5nlkOMlHrVDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T15:45:17.543844Z","bundle_sha256":"3823b61ba0eb305c6d505bd30f65e6f349ef80625f7c59951284809a2e1a44df"}}