{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:OI6QZSLIU4XW36FXU3QMCI3LD3","short_pith_number":"pith:OI6QZSLI","schema_version":"1.0","canonical_sha256":"723d0cc968a72f6df8b7a6e0c1236b1ee319a37cdcf4a2556454afa204ed9fac","source":{"kind":"arxiv","id":"2505.15146","version":2},"attestation_state":"computed","paper":{"title":"lmgame-Bench: How Good are LLMs at Playing Games?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Eric P. Xing, Haojian Jin, Haoyang Yu, Hao Zhang, Ion Stoica, Lanxiang Hu, Mingjia Huo, Tajana Rosing, Yuxuan Zhang","submitted_at":"2025-05-21T06:02:55Z","abstract_excerpt":"Playing video games requires perception, memory, and planning, exactly the faculties modern large language model (LLM) agents are expected to master. We study the major challenges in using popular video games to evaluate modern LLMs and find that directly dropping LLMs into games cannot make an effective evaluation, for three reasons -- brittle vision perception, prompt sensitivity, and potential data contamination. We introduce lmgame-Bench to turn games into reliable evaluations. lmgame-Bench features a suite of platformer, puzzle, and narrative games delivered through a unified Gym-style AP"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.15146","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-05-21T06:02:55Z","cross_cats_sorted":[],"title_canon_sha256":"5ea5e616461b067a5d222fed88b2e1aa45afe5edc0558eb173851d0ee163284a","abstract_canon_sha256":"39af4c16d010f54d48bc8c36a35b4c4f841e28bc627b7e2d70ba473e5ecbc660"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:14:59.922182Z","signature_b64":"BeuQzW58uqEeuDUIqiTtU2YN4uoYa7cvNdW0HxC54hvFvFcHyavyD+hmF5T3otmBMsMB71WmxOPaR84Nt9ZZAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"723d0cc968a72f6df8b7a6e0c1236b1ee319a37cdcf4a2556454afa204ed9fac","last_reissued_at":"2026-07-05T11:14:59.921689Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:14:59.921689Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"lmgame-Bench: How Good are LLMs at Playing Games?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Eric P. Xing, Haojian Jin, Haoyang Yu, Hao Zhang, Ion Stoica, Lanxiang Hu, Mingjia Huo, Tajana Rosing, Yuxuan Zhang","submitted_at":"2025-05-21T06:02:55Z","abstract_excerpt":"Playing video games requires perception, memory, and planning, exactly the faculties modern large language model (LLM) agents are expected to master. We study the major challenges in using popular video games to evaluate modern LLMs and find that directly dropping LLMs into games cannot make an effective evaluation, for three reasons -- brittle vision perception, prompt sensitivity, and potential data contamination. We introduce lmgame-Bench to turn games into reliable evaluations. lmgame-Bench features a suite of platformer, puzzle, and narrative games delivered through a unified Gym-style AP"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.15146","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.15146/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.15146","created_at":"2026-07-05T11:14:59.921748+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.15146v2","created_at":"2026-07-05T11:14:59.921748+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.15146","created_at":"2026-07-05T11:14:59.921748+00:00"},{"alias_kind":"pith_short_12","alias_value":"OI6QZSLIU4XW","created_at":"2026-07-05T11:14:59.921748+00:00"},{"alias_kind":"pith_short_16","alias_value":"OI6QZSLIU4XW36FX","created_at":"2026-07-05T11:14:59.921748+00:00"},{"alias_kind":"pith_short_8","alias_value":"OI6QZSLI","created_at":"2026-07-05T11:14:59.921748+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":19,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.18950","citing_title":"RTSGameBench: An RTS Benchmark for Strategic Reasoning by Vision-Language Models","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2606.19338","citing_title":"Beyond the Current Observation: Evaluating Multimodal Large Language Models in Controllable Non-Markov Games","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09826","citing_title":"OmniGameArena: A Unified UE5 Benchmark for VLM Game Agents with Improvement Dynamics","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2606.24893","citing_title":"AgentOdyssey: Open-Ended Long-Horizon Text Game Generation for Test-Time Continual Learning Agents","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13527","citing_title":"MMSkills: Towards Multimodal Skills for General Visual Agents","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29512","citing_title":"MINDGAMES: A Live Arena for Evaluating Social and Strategic Reasoning in Multi-Agent LLMs","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29653","citing_title":"PTCG-Bench: Can LLM Agents Master Pok\\'emon Trading Card Game?","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06556","citing_title":"Robots Need More than VLA and World Models","ref_index":152,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12191","citing_title":"Agentic Environment Engineering for Large Language Models: A Survey of Environment Modeling, Synthesis, Evaluation, and Application","ref_index":116,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17637","citing_title":"WebGameBench: Requirement-to-Application Evaluation for Coding Agents via Browser-Native Games","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17637","citing_title":"WebGameBench: Requirement-to-Application Evaluation for Coding Agents via Browser-Native Games","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2509.08827","citing_title":"A Survey of Reinforcement Learning for Large Reasoning Models","ref_index":203,"is_internal_anchor":false},{"citing_arxiv_id":"2603.15432","citing_title":"Gym-V: A Unified Vision Environment System for Agentic Vision Research","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13527","citing_title":"MMSkills: Towards Multimodal Skills for General Visual Agents","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13527","citing_title":"MMSkills: Towards Multimodal Skills for General Visual Agents","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2509.02544","citing_title":"UI-TARS-2 Technical Report: Advancing GUI Agent with Multi-Turn Reinforcement Learning","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00347","citing_title":"Odysseus: Scaling VLMs to 100+ Turn Decision-Making in Games via Reinforcement Learning","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2604.20043","citing_title":"TriEx: A Game-based Tri-View Framework for Explaining Internal Reasoning in Multi-Agent LLMs","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2604.20987","citing_title":"Co-Evolving LLM Decision and Skill Bank Agents for Long-Horizon Tasks","ref_index":7,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3","json":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3.json","graph_json":"https://pith.science/api/pith-number/OI6QZSLIU4XW36FXU3QMCI3LD3/graph.json","events_json":"https://pith.science/api/pith-number/OI6QZSLIU4XW36FXU3QMCI3LD3/events.json","paper":"https://pith.science/paper/OI6QZSLI"},"agent_actions":{"view_html":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3","download_json":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3.json","view_paper":"https://pith.science/paper/OI6QZSLI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.15146&json=true","fetch_graph":"https://pith.science/api/pith-number/OI6QZSLIU4XW36FXU3QMCI3LD3/graph.json","fetch_events":"https://pith.science/api/pith-number/OI6QZSLIU4XW36FXU3QMCI3LD3/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3/action/timestamp_anchor","attest_storage":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3/action/storage_attestation","attest_author":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3/action/author_attestation","sign_citation":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3/action/citation_signature","submit_replication":"https://pith.science/pith/OI6QZSLIU4XW36FXU3QMCI3LD3/action/replication_record"}},"created_at":"2026-07-05T11:14:59.921748+00:00","updated_at":"2026-07-05T11:14:59.921748+00:00"}