{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:FAJJCCRIZZHLD364M3CPX3RHZD","short_pith_number":"pith:FAJJCCRI","schema_version":"1.0","canonical_sha256":"2812910a28ce4eb1efdc66c4fbee27c8df62ac51c61e6bf6da3022335d775fff","source":{"kind":"arxiv","id":"2601.19924","version":2},"attestation_state":"computed","paper":{"title":"OPT-Engine: Benchmarking the Limits of LLMs in Optimization Modeling via Complexity Scaling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Solver-integrated LLMs for optimization modeling are limited primarily by errors in automated constraint formulation as problem complexity scales.","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Cheng cheng, Dongdong Ge, Yinan Sun, Yitian Chen, Zi Ling","submitted_at":"2026-01-09T09:22:33Z","abstract_excerpt":"We investigate the capabilities and scalability of Large Language Models (LLMs) in optimization modeling, a domain requiring structured reasoning and precise formulation. To this end, we introduce OPT-ENGINE, an extensible benchmark framework with quantifiable and controllable complexity. OPT-ENGINE spans ten canonical Operations Research problems, systematically scaling from Linear Programming to Mixed-Integer Programming, providing a structured environment to probe the limits of automated problem formulation and solving. Utilizing OPT-Engine, we address three pivotal research questions. Firs"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":false},"canonical_record":{"source":{"id":"2601.19924","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-01-09T09:22:33Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"c3847cfab4ea9627fcd570e0673fcc661a915ef3a464869b59c5cbfaed6149a7","abstract_canon_sha256":"9be0b4289a37c7b6d2030e8a8fde9772c80325d89a04fafa37c001f4001d4319"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:39:16.587836Z","signature_b64":"GEcomY8Bk3+Khk0S92E4+5g+qmD1gPoGINEGIIfOEyZta+ViLjFFBLQC0z148Tj4q+5KJpfZsGniS6V4oXW7BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2812910a28ce4eb1efdc66c4fbee27c8df62ac51c61e6bf6da3022335d775fff","last_reissued_at":"2026-05-17T23:39:16.587073Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:39:16.587073Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"OPT-Engine: Benchmarking the Limits of LLMs in Optimization Modeling via Complexity Scaling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Solver-integrated LLMs for optimization modeling are limited primarily by errors in automated constraint formulation as problem complexity scales.","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Cheng cheng, Dongdong Ge, Yinan Sun, Yitian Chen, Zi Ling","submitted_at":"2026-01-09T09:22:33Z","abstract_excerpt":"We investigate the capabilities and scalability of Large Language Models (LLMs) in optimization modeling, a domain requiring structured reasoning and precise formulation. To this end, we introduce OPT-ENGINE, an extensible benchmark framework with quantifiable and controllable complexity. OPT-ENGINE spans ten canonical Operations Research problems, systematically scaling from Linear Programming to Mixed-Integer Programming, providing a structured environment to probe the limits of automated problem formulation and solving. Utilizing OPT-Engine, we address three pivotal research questions. Firs"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"For the current SOTA paradigm, Solver-integrated Reasoning (SIR), the automated formulation of constraints represents the primary bottleneck.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"The assumption that the ten canonical problems and the chosen complexity scaling metrics (variables, constraints, integrality) sufficiently represent the space of real-world optimization modeling tasks that LLMs would encounter.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"OPT-Engine shows pure-text chain-of-thought reasoning in LLMs loses robustness as optimization complexity grows, external tools fix only local arithmetic, and solver-integrated methods are bottlenecked by automated constraint formulation.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Solver-integrated LLMs for optimization modeling are limited primarily by errors in automated constraint formulation as problem complexity scales.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"9c3fc2f0ceb5095e2a819ba56d33047d581664a6678ab39e105e890684cb7833"},"source":{"id":"2601.19924","kind":"arxiv","version":2},"verdict":{"id":"a9bcce02-5f71-486b-a47b-54af35ebb9b0","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T16:15:21.254376Z","strongest_claim":"For the current SOTA paradigm, Solver-integrated Reasoning (SIR), the automated formulation of constraints represents the primary bottleneck.","one_line_summary":"OPT-Engine shows pure-text chain-of-thought reasoning in LLMs loses robustness as optimization complexity grows, external tools fix only local arithmetic, and solver-integrated methods are bottlenecked by automated constraint formulation.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"The assumption that the ten canonical problems and the chosen complexity scaling metrics (variables, constraints, integrality) sufficiently represent the space of real-world optimization modeling tasks that LLMs would encounter.","pith_extraction_headline":"Solver-integrated LLMs for optimization modeling are limited primarily by errors in automated constraint formulation as problem complexity scales."},"references":{"count":43,"sample":[{"doi":"","year":2023,"title":"GPT-4 Technical Report","work_id":"b928e041-6991-4c08-8c81-0359e4097c7b","ref_index":1,"cited_arxiv_id":"2303.08774","is_internal_anchor":true},{"doi":"","year":2023,"title":"Gemini: A Family of Highly Capable Multimodal Models","work_id":"83f7c85b-3f11-450f-ac0c-64d9745220b2","ref_index":2,"cited_arxiv_id":"2312.11805","is_internal_anchor":true},{"doi":"","year":2024,"title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","work_id":"80e3e977-f1bb-4c83-8d0c-1ab0a0c5c3f1","ref_index":3,"cited_arxiv_id":"2403.05530","is_internal_anchor":true},{"doi":"","year":2024,"title":"DeepSeek-V3 Technical Report","work_id":"57d2791d-2219-4c31-a077-afc04b12a75c","ref_index":4,"cited_arxiv_id":"2412.19437","is_internal_anchor":true},{"doi":"","year":2025,"title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","work_id":"e6b75ad5-2877-4168-97c8-710407094d20","ref_index":5,"cited_arxiv_id":"2501.12948","is_internal_anchor":true}],"resolved_work":43,"snapshot_sha256":"2ce58239370396c817464bfce08651c77a096182468c96fcae0b0da971e819be","internal_anchors":12},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2601.19924","created_at":"2026-05-17T23:39:16.587208+00:00"},{"alias_kind":"arxiv_version","alias_value":"2601.19924v2","created_at":"2026-05-17T23:39:16.587208+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2601.19924","created_at":"2026-05-17T23:39:16.587208+00:00"},{"alias_kind":"pith_short_12","alias_value":"FAJJCCRIZZHL","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"FAJJCCRIZZHLD364","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"FAJJCCRI","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2604.25847","citing_title":"From Soliloquy to Agora: Memory-Enhanced LLM Agents with Decentralized Debate for Optimization Modeling","ref_index":3,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD","json":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD.json","graph_json":"https://pith.science/api/pith-number/FAJJCCRIZZHLD364M3CPX3RHZD/graph.json","events_json":"https://pith.science/api/pith-number/FAJJCCRIZZHLD364M3CPX3RHZD/events.json","paper":"https://pith.science/paper/FAJJCCRI"},"agent_actions":{"view_html":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD","download_json":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD.json","view_paper":"https://pith.science/paper/FAJJCCRI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2601.19924&json=true","fetch_graph":"https://pith.science/api/pith-number/FAJJCCRIZZHLD364M3CPX3RHZD/graph.json","fetch_events":"https://pith.science/api/pith-number/FAJJCCRIZZHLD364M3CPX3RHZD/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD/action/storage_attestation","attest_author":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD/action/author_attestation","sign_citation":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD/action/citation_signature","submit_replication":"https://pith.science/pith/FAJJCCRIZZHLD364M3CPX3RHZD/action/replication_record"}},"created_at":"2026-05-17T23:39:16.587208+00:00","updated_at":"2026-05-17T23:39:16.587208+00:00"}