{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LGLZBBSUS26O6CMRN7KSPLUTQN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f46cf919706d875f4480a757a35e23fa7b95da7e7cfda3053abede483922f4ba","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-01T17:10:35Z","title_canon_sha256":"2163f390390ba469ea48dc253268f9d39b104ad277229e9583fcfd6feff9c179"},"schema_version":"1.0","source":{"id":"2402.00782","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.00782","created_at":"2026-07-05T07:40:17Z"},{"alias_kind":"arxiv_version","alias_value":"2402.00782v1","created_at":"2026-07-05T07:40:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.00782","created_at":"2026-07-05T07:40:17Z"},{"alias_kind":"pith_short_12","alias_value":"LGLZBBSUS26O","created_at":"2026-07-05T07:40:17Z"},{"alias_kind":"pith_short_16","alias_value":"LGLZBBSUS26O6CMR","created_at":"2026-07-05T07:40:17Z"},{"alias_kind":"pith_short_8","alias_value":"LGLZBBSU","created_at":"2026-07-05T07:40:17Z"}],"graph_snapshots":[{"event_id":"sha256:6e0f2d3b09e3f1e600fd00458dc8e7b8d303052c17181c5a61cc3f2a225e193d","target":"graph","created_at":"2026-07-05T07:40:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.00782/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has been credited as the key advance that has allowed Large Language Models (LLMs) to effectively follow instructions and produce useful assistance. Classically, this involves generating completions from the LLM in response to a query before using a separate reward model to assign a score to the full completion. As an auto-regressive process, the LLM has to take many \"actions\" (selecting individual tokens) and only receives a single, sparse reward at the end of an episode, a setup that is known to be difficult to optimise in traditional reinfor","authors_text":"Alex J. Chan, Hao Sun, Mihaela van der Schaar, Samuel Holt","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-01T17:10:35Z","title":"Dense Reward for Free in Reinforcement Learning from Human Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.00782","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:24bb7b1291f696b93574f71365a5c285c06524b7e835f9aeb154ca77a4851f6e","target":"record","created_at":"2026-07-05T07:40:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f46cf919706d875f4480a757a35e23fa7b95da7e7cfda3053abede483922f4ba","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-02-01T17:10:35Z","title_canon_sha256":"2163f390390ba469ea48dc253268f9d39b104ad277229e9583fcfd6feff9c179"},"schema_version":"1.0","source":{"id":"2402.00782","kind":"arxiv","version":1}},"canonical_sha256":"599790865496bcef09916fd527ae93834a13dd979fa74d0381c661ec02bd9943","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"599790865496bcef09916fd527ae93834a13dd979fa74d0381c661ec02bd9943","first_computed_at":"2026-07-05T07:40:17.617458Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:40:17.617458Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6ykQwH2NvhXpXBSeQHpAcljSnQXGeSbrHaqnJkF4MU0Atkqp2hwl9Qp+FGi9rgnxvje5XPcN07+Z4OGr5E3CDg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:40:17.617871Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.00782","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:24bb7b1291f696b93574f71365a5c285c06524b7e835f9aeb154ca77a4851f6e","sha256:6e0f2d3b09e3f1e600fd00458dc8e7b8d303052c17181c5a61cc3f2a225e193d"],"state_sha256":"2a746fd5b2170dba7bfeef173767a8ce3590cc12ca15cbfe7a14da5ebc6474a0"}