{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:HKG6ZIF6M62J4TFERT5V7GPAHI","short_pith_number":"pith:HKG6ZIF6","schema_version":"1.0","canonical_sha256":"3a8deca0be67b49e4ca48cfb5f99e03a0b4fd840164a7b78a626b631f8746b41","source":{"kind":"arxiv","id":"2402.03282","version":3},"attestation_state":"computed","paper":{"title":"A Theoretical Framework for Partially Observed Reward-States in RLHF","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Aldo Pacchiano, Ambuj Tewari, Chinmaya Kausik, Mirco Mutti","submitted_at":"2024-02-05T18:38:55Z","abstract_excerpt":"The growing deployment of reinforcement learning from human feedback (RLHF) calls for a deeper theoretical investigation of its underlying models. The prevalent models of RLHF do not account for neuroscience-backed, partially-observed \"internal states\" that can affect human feedback, nor do they accommodate intermediate feedback during an interaction. Both of these can be instrumental in speeding up learning and improving alignment. To address these limitations, we model RLHF as reinforcement learning with partially observed reward-states (PORRL). We accommodate two kinds of feedback $-$ cardi"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.03282","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-05T18:38:55Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"b381542f1279c16429d2c5882092d501fdee609005badc6837290f45dc8df1dd","abstract_canon_sha256":"970e1734c6b792b0535d8b1ba55b99a9b30c560b692305e8ddee2b2898e9d509"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:33:08.079236Z","signature_b64":"9oWkQgywwS3QJ4iwRLQvUOkOyxyb5HbaOk2KEdOjA/yIJASN0kf+kb/67udTkLJyqz1rKiOwqVzWF3/l1GPcDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3a8deca0be67b49e4ca48cfb5f99e03a0b4fd840164a7b78a626b631f8746b41","last_reissued_at":"2026-07-05T09:33:08.078630Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:33:08.078630Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Theoretical Framework for Partially Observed Reward-States in RLHF","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Aldo Pacchiano, Ambuj Tewari, Chinmaya Kausik, Mirco Mutti","submitted_at":"2024-02-05T18:38:55Z","abstract_excerpt":"The growing deployment of reinforcement learning from human feedback (RLHF) calls for a deeper theoretical investigation of its underlying models. The prevalent models of RLHF do not account for neuroscience-backed, partially-observed \"internal states\" that can affect human feedback, nor do they accommodate intermediate feedback during an interaction. Both of these can be instrumental in speeding up learning and improving alignment. To address these limitations, we model RLHF as reinforcement learning with partially observed reward-states (PORRL). We accommodate two kinds of feedback $-$ cardi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.03282","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.03282/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.03282","created_at":"2026-07-05T09:33:08.078702+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.03282v3","created_at":"2026-07-05T09:33:08.078702+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.03282","created_at":"2026-07-05T09:33:08.078702+00:00"},{"alias_kind":"pith_short_12","alias_value":"HKG6ZIF6M62J","created_at":"2026-07-05T09:33:08.078702+00:00"},{"alias_kind":"pith_short_16","alias_value":"HKG6ZIF6M62J4TFE","created_at":"2026-07-05T09:33:08.078702+00:00"},{"alias_kind":"pith_short_8","alias_value":"HKG6ZIF6","created_at":"2026-07-05T09:33:08.078702+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.30271","citing_title":"Preference-Based Reward Learning under Partial Observability with Inexact Dynamics","ref_index":10,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI","json":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI.json","graph_json":"https://pith.science/api/pith-number/HKG6ZIF6M62J4TFERT5V7GPAHI/graph.json","events_json":"https://pith.science/api/pith-number/HKG6ZIF6M62J4TFERT5V7GPAHI/events.json","paper":"https://pith.science/paper/HKG6ZIF6"},"agent_actions":{"view_html":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI","download_json":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI.json","view_paper":"https://pith.science/paper/HKG6ZIF6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.03282&json=true","fetch_graph":"https://pith.science/api/pith-number/HKG6ZIF6M62J4TFERT5V7GPAHI/graph.json","fetch_events":"https://pith.science/api/pith-number/HKG6ZIF6M62J4TFERT5V7GPAHI/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI/action/timestamp_anchor","attest_storage":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI/action/storage_attestation","attest_author":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI/action/author_attestation","sign_citation":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI/action/citation_signature","submit_replication":"https://pith.science/pith/HKG6ZIF6M62J4TFERT5V7GPAHI/action/replication_record"}},"created_at":"2026-07-05T09:33:08.078702+00:00","updated_at":"2026-07-05T09:33:08.078702+00:00"}