{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:VKZX2R4K6SQZGJCE7XAEPSESL3","short_pith_number":"pith:VKZX2R4K","schema_version":"1.0","canonical_sha256":"aab37d478af4a1932444fdc047c8925ee45e086a60f644edd755c275344fc9d5","source":{"kind":"arxiv","id":"2408.14874","version":2},"attestation_state":"computed","paper":{"title":"Inverse-Q*: Token Level Reinforcement Learning for Aligning Large Language Models Without Preference Data","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Han Xia, Qiming Ge, Qi Zhang, Songyang Gao, Xuanjing Huang, Zhiheng Xi","submitted_at":"2024-08-27T08:43:32Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has proven effective in aligning large language models with human intentions, yet it often relies on complex methodologies like Proximal Policy Optimization (PPO) that require extensive hyper-parameter tuning and present challenges in sample efficiency and stability. In this paper, we introduce Inverse-Q*, an innovative framework that transcends traditional RL methods by optimizing token-level reinforcement learning without the need for additional reward or value models. Inverse-Q* leverages direct preference optimization techniques but extends"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.14874","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-27T08:43:32Z","cross_cats_sorted":[],"title_canon_sha256":"c0ff60432454e1356e6f465f42137d6ae3aaa3728520466c3b38869c05036634","abstract_canon_sha256":"e26f5a21d0f00abdc0fbcc52a39047773121b9f36ee6658b157f6a421e3152e1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:00:38.848738Z","signature_b64":"qSsz3FPoudNTFgSGo+AeprIA7VPu0K39UZv7KVByoQfAXNNs7h5EMfmOH48tDAtnFnVRUznZm6eX8jHZnBV8Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"aab37d478af4a1932444fdc047c8925ee45e086a60f644edd755c275344fc9d5","last_reissued_at":"2026-07-05T09:00:38.848266Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:00:38.848266Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Inverse-Q*: Token Level Reinforcement Learning for Aligning Large Language Models Without Preference Data","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Han Xia, Qiming Ge, Qi Zhang, Songyang Gao, Xuanjing Huang, Zhiheng Xi","submitted_at":"2024-08-27T08:43:32Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has proven effective in aligning large language models with human intentions, yet it often relies on complex methodologies like Proximal Policy Optimization (PPO) that require extensive hyper-parameter tuning and present challenges in sample efficiency and stability. In this paper, we introduce Inverse-Q*, an innovative framework that transcends traditional RL methods by optimizing token-level reinforcement learning without the need for additional reward or value models. Inverse-Q* leverages direct preference optimization techniques but extends"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.14874","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.14874/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.14874","created_at":"2026-07-05T09:00:38.848327+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.14874v2","created_at":"2026-07-05T09:00:38.848327+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.14874","created_at":"2026-07-05T09:00:38.848327+00:00"},{"alias_kind":"pith_short_12","alias_value":"VKZX2R4K6SQZ","created_at":"2026-07-05T09:00:38.848327+00:00"},{"alias_kind":"pith_short_16","alias_value":"VKZX2R4K6SQZGJCE","created_at":"2026-07-05T09:00:38.848327+00:00"},{"alias_kind":"pith_short_8","alias_value":"VKZX2R4K","created_at":"2026-07-05T09:00:38.848327+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3","json":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3.json","graph_json":"https://pith.science/api/pith-number/VKZX2R4K6SQZGJCE7XAEPSESL3/graph.json","events_json":"https://pith.science/api/pith-number/VKZX2R4K6SQZGJCE7XAEPSESL3/events.json","paper":"https://pith.science/paper/VKZX2R4K"},"agent_actions":{"view_html":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3","download_json":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3.json","view_paper":"https://pith.science/paper/VKZX2R4K","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.14874&json=true","fetch_graph":"https://pith.science/api/pith-number/VKZX2R4K6SQZGJCE7XAEPSESL3/graph.json","fetch_events":"https://pith.science/api/pith-number/VKZX2R4K6SQZGJCE7XAEPSESL3/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3/action/timestamp_anchor","attest_storage":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3/action/storage_attestation","attest_author":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3/action/author_attestation","sign_citation":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3/action/citation_signature","submit_replication":"https://pith.science/pith/VKZX2R4K6SQZGJCE7XAEPSESL3/action/replication_record"}},"created_at":"2026-07-05T09:00:38.848327+00:00","updated_at":"2026-07-05T09:00:38.848327+00:00"}