{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:F2T337YLPEJNMS743KI556MG6L","short_pith_number":"pith:F2T337YL","schema_version":"1.0","canonical_sha256":"2ea7bdff0b7912d64bfcda91def986f2d89a36dbd573db3663d57de8dead561c","source":{"kind":"arxiv","id":"2503.19595","version":2},"attestation_state":"computed","paper":{"title":"Optimizing Language Models for Inference Time Objectives using Reinforcement Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Gabriel Synnaeve, Kunhao Zheng, R\\'emi Munos, Yunhao Tang","submitted_at":"2025-03-25T12:21:26Z","abstract_excerpt":"In this work, we investigate the merits of explicitly optimizing for inference time algorithmic performance during model training. We show how optimizing for inference time performance can improve overall model efficacy. We consider generic inference time objectives with $k$ samples, with a focus on pass@$k$ and majority voting as two main applications. With language model training on reasoning datasets, we showcase the performance trade-off enabled by training with such objectives. When training on code generation tasks, we show that the approach significantly improves pass@$k$ objectives com"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2503.19595","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:21:26Z","cross_cats_sorted":[],"title_canon_sha256":"710b9c14f126806fbc3afe50962750e496e34a7926ccabfb2ed1707e6cfb853b","abstract_canon_sha256":"40aad8901f3d8c0d090e4e090206ef69709c6e13706873d377d3e2758c7a7293"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:54:45.691575Z","signature_b64":"R2GqmcWR0i2o1ZeIU6PitYGjoE4UP4kspzXqTvzW9wnzw9Q8YIopsaZo3tcYszhA6SO/JOqkuDX1D+mLu1wfCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2ea7bdff0b7912d64bfcda91def986f2d89a36dbd573db3663d57de8dead561c","last_reissued_at":"2026-07-05T11:54:45.691052Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:54:45.691052Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Optimizing Language Models for Inference Time Objectives using Reinforcement Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Gabriel Synnaeve, Kunhao Zheng, R\\'emi Munos, Yunhao Tang","submitted_at":"2025-03-25T12:21:26Z","abstract_excerpt":"In this work, we investigate the merits of explicitly optimizing for inference time algorithmic performance during model training. We show how optimizing for inference time performance can improve overall model efficacy. We consider generic inference time objectives with $k$ samples, with a focus on pass@$k$ and majority voting as two main applications. With language model training on reasoning datasets, we showcase the performance trade-off enabled by training with such objectives. When training on code generation tasks, we show that the approach significantly improves pass@$k$ objectives com"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.19595","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.19595/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2503.19595","created_at":"2026-07-05T11:54:45.691110+00:00"},{"alias_kind":"arxiv_version","alias_value":"2503.19595v2","created_at":"2026-07-05T11:54:45.691110+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.19595","created_at":"2026-07-05T11:54:45.691110+00:00"},{"alias_kind":"pith_short_12","alias_value":"F2T337YLPEJN","created_at":"2026-07-05T11:54:45.691110+00:00"},{"alias_kind":"pith_short_16","alias_value":"F2T337YLPEJNMS74","created_at":"2026-07-05T11:54:45.691110+00:00"},{"alias_kind":"pith_short_8","alias_value":"F2T337YL","created_at":"2026-07-05T11:54:45.691110+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.23595","citing_title":"SPIRAL: Learning to Search and Aggregate","ref_index":48,"is_internal_anchor":false},{"citing_arxiv_id":"2606.18910","citing_title":"REVES: REvision and VErification--Augmented Training for Test-Time Scaling","ref_index":67,"is_internal_anchor":false},{"citing_arxiv_id":"2607.01490","citing_title":"Don't Let Gains FADE: Breaking Down Policy Gradient Weights in RL","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2607.02390","citing_title":"DecompRL: Solving Harder Problems by Learning Modular Code Generation","ref_index":55,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24957","citing_title":"Compute Aligned Training: Optimizing for Test Time Inference","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20854","citing_title":"Finite-Time Regret Analysis of Retry-Aware Bandits","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2509.25424","citing_title":"Polychromic Objectives for Reinforcement Learning","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10716","citing_title":"What should post-training optimize? A test-time scaling law perspective","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24957","citing_title":"Compute Aligned Training: Optimizing for Test Time Inference","ref_index":13,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L","json":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L.json","graph_json":"https://pith.science/api/pith-number/F2T337YLPEJNMS743KI556MG6L/graph.json","events_json":"https://pith.science/api/pith-number/F2T337YLPEJNMS743KI556MG6L/events.json","paper":"https://pith.science/paper/F2T337YL"},"agent_actions":{"view_html":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L","download_json":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L.json","view_paper":"https://pith.science/paper/F2T337YL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2503.19595&json=true","fetch_graph":"https://pith.science/api/pith-number/F2T337YLPEJNMS743KI556MG6L/graph.json","fetch_events":"https://pith.science/api/pith-number/F2T337YLPEJNMS743KI556MG6L/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L/action/timestamp_anchor","attest_storage":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L/action/storage_attestation","attest_author":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L/action/author_attestation","sign_citation":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L/action/citation_signature","submit_replication":"https://pith.science/pith/F2T337YLPEJNMS743KI556MG6L/action/replication_record"}},"created_at":"2026-07-05T11:54:45.691110+00:00","updated_at":"2026-07-05T11:54:45.691110+00:00"}