{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VAG54AWAQLBKOGM6WZK4VN3QNL","short_pith_number":"pith:VAG54AWA","canonical_record":{"source":{"id":"2411.11761","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-18T17:40:42Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"569c4ef50b5db4e3ac9fa0b56c51f08ef1ac7c98c323215fb0f0aae4cc68aee9","abstract_canon_sha256":"1b98aa541ff6009fd3db7475e880203fd2d415195daf68a55c4e6c37644a7370"},"schema_version":"1.0"},"canonical_sha256":"a80dde02c082c2a7199eb655cab7706ac7103f483dee7d7f7b80acf19bc5eddc","source":{"kind":"arxiv","id":"2411.11761","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.11761","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"arxiv_version","alias_value":"2411.11761v2","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.11761","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_12","alias_value":"VAG54AWAQLBK","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_16","alias_value":"VAG54AWAQLBKOGM6","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_8","alias_value":"VAG54AWA","created_at":"2026-07-05T10:17:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VAG54AWAQLBKOGM6WZK4VN3QNL","target":"record","payload":{"canonical_record":{"source":{"id":"2411.11761","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-18T17:40:42Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"569c4ef50b5db4e3ac9fa0b56c51f08ef1ac7c98c323215fb0f0aae4cc68aee9","abstract_canon_sha256":"1b98aa541ff6009fd3db7475e880203fd2d415195daf68a55c4e6c37644a7370"},"schema_version":"1.0"},"canonical_sha256":"a80dde02c082c2a7199eb655cab7706ac7103f483dee7d7f7b80acf19bc5eddc","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:17:13.879011Z","signature_b64":"K6BpAByEOWa2i1guhUc7sWq/ZE2zgJBQ80yzNYxEYjw90ff0Rb0pWej+3xUy+8CYpadG+RTjnpC2d+d4T4pcAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a80dde02c082c2a7199eb655cab7706ac7103f483dee7d7f7b80acf19bc5eddc","last_reissued_at":"2026-07-05T10:17:13.878526Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:17:13.878526Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.11761","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2s8ad0PqvJExUbjEm2JoxMSmMIaB1uSNjlLL+HOEgLuVPQ/HVJ9SztW8TpDNFraVXPJthMgD/NuT12+sKOXTAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T04:56:10.004210Z"},"content_sha256":"32260509786b901da7fa356aa9437fca187284804ab4ad2fd2bf96c350c1daf4","schema_version":"1.0","event_id":"sha256:32260509786b901da7fa356aa9437fca187284804ab4ad2fd2bf96c350c1daf4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VAG54AWAQLBKOGM6WZK4VN3QNL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Mapping out the Space of Human Feedback for Reinforcement Learning: A Conceptual Framework","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.LG","authors_text":"David Lindner, Mennatallah El-Assady, Rapha\\\"el Baur, Yannick Metz","submitted_at":"2024-11-18T17:40:42Z","abstract_excerpt":"Reinforcement Learning from Human feedback (RLHF) has become a powerful tool to fine-tune or train agentic machine learning models. Similar to how humans interact in social contexts, we can use many types of feedback to communicate our preferences, intentions, and knowledge to an RL agent. However, applications of human feedback in RL are often limited in scope and disregard human factors. In this work, we bridge the gap between machine learning and human-computer interaction efforts by developing a shared understanding of human feedback in interactive learning scenarios. We first introduce a "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.11761","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.11761/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mhiTrRIL6Waklp2BupfOZn2ejvzXoblKFzNQO6b6QHsx0jda4b4pvu8AHt/lGxr6CUvCvLFRvLOtI5xmFDV0BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T04:56:10.005078Z"},"content_sha256":"6b6a5df138e4277b37221bfd31b5e920935608c9c7f64d07609226baafdf25e4","schema_version":"1.0","event_id":"sha256:6b6a5df138e4277b37221bfd31b5e920935608c9c7f64d07609226baafdf25e4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/bundle.json","state_url":"https://pith.science/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T04:56:10Z","links":{"resolver":"https://pith.science/pith/VAG54AWAQLBKOGM6WZK4VN3QNL","bundle":"https://pith.science/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/bundle.json","state":"https://pith.science/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VAG54AWAQLBKOGM6WZK4VN3QNL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VAG54AWAQLBKOGM6WZK4VN3QNL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1b98aa541ff6009fd3db7475e880203fd2d415195daf68a55c4e6c37644a7370","cross_cats_sorted":["cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-18T17:40:42Z","title_canon_sha256":"569c4ef50b5db4e3ac9fa0b56c51f08ef1ac7c98c323215fb0f0aae4cc68aee9"},"schema_version":"1.0","source":{"id":"2411.11761","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.11761","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"arxiv_version","alias_value":"2411.11761v2","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.11761","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_12","alias_value":"VAG54AWAQLBK","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_16","alias_value":"VAG54AWAQLBKOGM6","created_at":"2026-07-05T10:17:13Z"},{"alias_kind":"pith_short_8","alias_value":"VAG54AWA","created_at":"2026-07-05T10:17:13Z"}],"graph_snapshots":[{"event_id":"sha256:6b6a5df138e4277b37221bfd31b5e920935608c9c7f64d07609226baafdf25e4","target":"graph","created_at":"2026-07-05T10:17:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.11761/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human feedback (RLHF) has become a powerful tool to fine-tune or train agentic machine learning models. Similar to how humans interact in social contexts, we can use many types of feedback to communicate our preferences, intentions, and knowledge to an RL agent. However, applications of human feedback in RL are often limited in scope and disregard human factors. In this work, we bridge the gap between machine learning and human-computer interaction efforts by developing a shared understanding of human feedback in interactive learning scenarios. We first introduce a ","authors_text":"David Lindner, Mennatallah El-Assady, Rapha\\\"el Baur, Yannick Metz","cross_cats":["cs.HC"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-18T17:40:42Z","title":"Mapping out the Space of Human Feedback for Reinforcement Learning: A Conceptual Framework"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.11761","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:32260509786b901da7fa356aa9437fca187284804ab4ad2fd2bf96c350c1daf4","target":"record","created_at":"2026-07-05T10:17:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1b98aa541ff6009fd3db7475e880203fd2d415195daf68a55c4e6c37644a7370","cross_cats_sorted":["cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-18T17:40:42Z","title_canon_sha256":"569c4ef50b5db4e3ac9fa0b56c51f08ef1ac7c98c323215fb0f0aae4cc68aee9"},"schema_version":"1.0","source":{"id":"2411.11761","kind":"arxiv","version":2}},"canonical_sha256":"a80dde02c082c2a7199eb655cab7706ac7103f483dee7d7f7b80acf19bc5eddc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a80dde02c082c2a7199eb655cab7706ac7103f483dee7d7f7b80acf19bc5eddc","first_computed_at":"2026-07-05T10:17:13.878526Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:17:13.878526Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"K6BpAByEOWa2i1guhUc7sWq/ZE2zgJBQ80yzNYxEYjw90ff0Rb0pWej+3xUy+8CYpadG+RTjnpC2d+d4T4pcAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:17:13.879011Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.11761","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:32260509786b901da7fa356aa9437fca187284804ab4ad2fd2bf96c350c1daf4","sha256:6b6a5df138e4277b37221bfd31b5e920935608c9c7f64d07609226baafdf25e4"],"state_sha256":"8ca399257bc47afc2949861931ccc2a5759e4edbbde4aa0d829a8ee8f78923a1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HmMecHa568q3jHQMetR1mmMVmZif7qR3kFcIv9bjfOr2wvZjBev+vSIKixEOYha3i08BTsRbUaCSFSao/0+XAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T04:56:10.013191Z","bundle_sha256":"3165d03315a26f0a830dc10bca9e6c1dcccc4aaf701d85fe096c074a1521f28c"}}