{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:MURRQ7E7MFITX7SSXY4IXZAUDL","short_pith_number":"pith:MURRQ7E7","schema_version":"1.0","canonical_sha256":"6523187c9f61513bfe52be388be4141acc9fb012c08e14683b0a7310159e4346","source":{"kind":"arxiv","id":"2506.17219","version":2},"attestation_state":"computed","paper":{"title":"No Free Lunch: Rethinking Internal Feedback for LLM Reasoning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Chen Wang, Haoxiang Guan, Jiyan He, Shuxin Zheng, Yanzhi Zhang, Yilin Cheng, Yitong Duan, Yue Wang, Zhaoxi Zhang","submitted_at":"2025-06-20T17:59:52Z","abstract_excerpt":"Reinforcement learning has emerged as a powerful paradigm for post-training large language models (LLMs) to improve reasoning. Approaches like Reinforcement Learning from Human Feedback (RLHF) and Reinforcement Learning with Verifiable Rewards (RLVR) have shown strong results, but they require extensive external supervision. We investigate an alternative class of methods, Reinforcement Learning from Internal Feedback (RLIF), which relies solely on intrinsic model-derived signals instead of external rewards. In particular, we leverage unsupervised reward proxies such as token-level entropy, tra"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.17219","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-06-20T17:59:52Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6b8d74f1b4dc4771eb1d63a620885501ef5467749fdf6a21656873a9f65d9777","abstract_canon_sha256":"8dfa62e1dac985fcd404c1c6ed1d76156ed71e011f6d0f74acabd2f99993731e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:26:53.204550Z","signature_b64":"KagncusLy9A4U8FW1vH6JhQc3y2/rVJdYcuvhDO6XVGEqW81Dsqi/zvOhjrAraKPE4JNJhiqo9u/Jlhdh/zmDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6523187c9f61513bfe52be388be4141acc9fb012c08e14683b0a7310159e4346","last_reissued_at":"2026-07-05T11:26:53.204077Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:26:53.204077Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"No Free Lunch: Rethinking Internal Feedback for LLM Reasoning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Chen Wang, Haoxiang Guan, Jiyan He, Shuxin Zheng, Yanzhi Zhang, Yilin Cheng, Yitong Duan, Yue Wang, Zhaoxi Zhang","submitted_at":"2025-06-20T17:59:52Z","abstract_excerpt":"Reinforcement learning has emerged as a powerful paradigm for post-training large language models (LLMs) to improve reasoning. Approaches like Reinforcement Learning from Human Feedback (RLHF) and Reinforcement Learning with Verifiable Rewards (RLVR) have shown strong results, but they require extensive external supervision. We investigate an alternative class of methods, Reinforcement Learning from Internal Feedback (RLIF), which relies solely on intrinsic model-derived signals instead of external rewards. In particular, we leverage unsupervised reward proxies such as token-level entropy, tra"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.17219","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.17219/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.17219","created_at":"2026-07-05T11:26:53.204133+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.17219v2","created_at":"2026-07-05T11:26:53.204133+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.17219","created_at":"2026-07-05T11:26:53.204133+00:00"},{"alias_kind":"pith_short_12","alias_value":"MURRQ7E7MFIT","created_at":"2026-07-05T11:26:53.204133+00:00"},{"alias_kind":"pith_short_16","alias_value":"MURRQ7E7MFITX7SS","created_at":"2026-07-05T11:26:53.204133+00:00"},{"alias_kind":"pith_short_8","alias_value":"MURRQ7E7","created_at":"2026-07-05T11:26:53.204133+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":11,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.18961","citing_title":"Be Your Own Teacher: Steering Protein Language Models via Unsupervised Reward Optimization","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2606.17803","citing_title":"Continual Self-Improvement with Lightweight Experiential Latent Memories","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04516","citing_title":"GeoMin: Data-Efficient Semi-Supervised RLVR via Geometric Distribution Modeling","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01249","citing_title":"Trust Region On-Policy Distillation","ref_index":83,"is_internal_anchor":false},{"citing_arxiv_id":"2602.12579","citing_title":"VI-CuRL: Stabilizing Verifier-Independent RL Reasoning via Confidence-Guided Variance Reduction","ref_index":37,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19444","citing_title":"Detecting and Mitigating the Correct-Answer Extinction Window in Test-Time Reinforcement Learning with Majority Voting","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17037","citing_title":"D$^2$Evo: Dual Difficulty-Aware Self-Evolution for Data-Efficient Reinforcement Learning","ref_index":59,"is_internal_anchor":false},{"citing_arxiv_id":"2509.05489","citing_title":"Self-Aligned Reward: Towards Effective and Efficient Reasoners","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04065","citing_title":"Free Energy-Driven Reinforcement Learning with Adaptive Advantage Shaping for Unsupervised Reasoning in LLMs","ref_index":62,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07244","citing_title":"Experience Sharing in Mutual Reinforcement Learning for Heterogeneous Language Models","ref_index":114,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17501","citing_title":"CoAct: Co-Active LLM Preference Learning with Human-AI Synergy","ref_index":7,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL","json":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL.json","graph_json":"https://pith.science/api/pith-number/MURRQ7E7MFITX7SSXY4IXZAUDL/graph.json","events_json":"https://pith.science/api/pith-number/MURRQ7E7MFITX7SSXY4IXZAUDL/events.json","paper":"https://pith.science/paper/MURRQ7E7"},"agent_actions":{"view_html":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL","download_json":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL.json","view_paper":"https://pith.science/paper/MURRQ7E7","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.17219&json=true","fetch_graph":"https://pith.science/api/pith-number/MURRQ7E7MFITX7SSXY4IXZAUDL/graph.json","fetch_events":"https://pith.science/api/pith-number/MURRQ7E7MFITX7SSXY4IXZAUDL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL/action/storage_attestation","attest_author":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL/action/author_attestation","sign_citation":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL/action/citation_signature","submit_replication":"https://pith.science/pith/MURRQ7E7MFITX7SSXY4IXZAUDL/action/replication_record"}},"created_at":"2026-07-05T11:26:53.204133+00:00","updated_at":"2026-07-05T11:26:53.204133+00:00"}