{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:LVIF2CDCVEMAT56A6MVCSEPCWN","short_pith_number":"pith:LVIF2CDC","schema_version":"1.0","canonical_sha256":"5d505d0862a91809f7c0f32a2911e2b37ff0e618dbc3ae2821a3b72f1d8147d2","source":{"kind":"arxiv","id":"2503.14499","version":4},"attestation_state":"computed","paper":{"title":"Measuring AI Ability to Complete Long Software Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Amy Deng, Ben West, Brian Goodrich, Chris Painter, Daniel M. Ziegler, David Rein, Elizabeth Barnes, Haoxing Du, Hjalmar Wijk, Joel Becker, Katharyn Garcia, Lawrence Chan, Lucas Jun Koba Sato, Luke Harold Miles, Max Hasin, Megan Kinniment, Nate Rush, Neev Parikh, Nikola Jurkovic, Ryan Bloom, Sami Jawhar, Seraphina Nix, Sydney von Arx, Tao Lin, Thomas Broadley, Thomas Kwa","submitted_at":"2025-03-18T17:59:31Z","abstract_excerpt":"Despite rapid progress on AI benchmarks, the real-world meaning of benchmark performance remains unclear. To quantify the capabilities of AI systems in terms of human capabilities, we propose a new metric: 50%-task-completion time horizon. This is the time humans typically take to complete tasks that AI models can complete with 50% success rate. We first timed humans with relevant domain expertise on a combination of RE-Bench, HCAST, and 66 novel shorter tasks. On these tasks, current frontier AI models such as Claude 3.7 Sonnet have a 50% time horizon of around 50 minutes. Furthermore, fronti"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2503.14499","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-18T17:59:31Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"bc9b28b5968b80a4075a2ad5e4e139cc5c7d55e38872bcdda7135c58a6c4fe2f","abstract_canon_sha256":"e702d008f561eedce194021837e05019ec617f99f259f86da88efe27237db942"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-14T00:18:18.549230Z","signature_b64":"2QNV+Z26UrJsfUQtMrAcSZYkeNWz0L1e9PqaV3B6BpuU7OUp+3YkaUyQNmc+s+JEi/OEab65a/xIr8KYUFtUDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5d505d0862a91809f7c0f32a2911e2b37ff0e618dbc3ae2821a3b72f1d8147d2","last_reissued_at":"2026-07-14T00:18:18.548212Z","signature_status":"signed_v1","first_computed_at":"2026-07-14T00:18:18.548212Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Measuring AI Ability to Complete Long Software Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Amy Deng, Ben West, Brian Goodrich, Chris Painter, Daniel M. Ziegler, David Rein, Elizabeth Barnes, Haoxing Du, Hjalmar Wijk, Joel Becker, Katharyn Garcia, Lawrence Chan, Lucas Jun Koba Sato, Luke Harold Miles, Max Hasin, Megan Kinniment, Nate Rush, Neev Parikh, Nikola Jurkovic, Ryan Bloom, Sami Jawhar, Seraphina Nix, Sydney von Arx, Tao Lin, Thomas Broadley, Thomas Kwa","submitted_at":"2025-03-18T17:59:31Z","abstract_excerpt":"Despite rapid progress on AI benchmarks, the real-world meaning of benchmark performance remains unclear. To quantify the capabilities of AI systems in terms of human capabilities, we propose a new metric: 50%-task-completion time horizon. This is the time humans typically take to complete tasks that AI models can complete with 50% success rate. We first timed humans with relevant domain expertise on a combination of RE-Bench, HCAST, and 66 novel shorter tasks. On these tasks, current frontier AI models such as Claude 3.7 Sonnet have a 50% time horizon of around 50 minutes. Furthermore, fronti"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.14499","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.14499/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2503.14499","created_at":"2026-07-14T00:18:18.548681+00:00"},{"alias_kind":"arxiv_version","alias_value":"2503.14499v4","created_at":"2026-07-14T00:18:18.548681+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.14499","created_at":"2026-07-14T00:18:18.548681+00:00"},{"alias_kind":"pith_short_12","alias_value":"LVIF2CDCVEMA","created_at":"2026-07-14T00:18:18.548681+00:00"},{"alias_kind":"pith_short_16","alias_value":"LVIF2CDCVEMAT56A","created_at":"2026-07-14T00:18:18.548681+00:00"},{"alias_kind":"pith_short_8","alias_value":"LVIF2CDC","created_at":"2026-07-14T00:18:18.548681+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":47,"internal_anchor_count":47,"sample":[{"citing_arxiv_id":"2607.08066","citing_title":"Persuasion Attacks Can Decrease Effectiveness of CoT Monitoring","ref_index":93,"is_internal_anchor":true},{"citing_arxiv_id":"2607.06411","citing_title":"RuBench: A Repository-Level Agentic Coding Benchmark with Natively Authored Russian Task Specifications","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2606.00036","citing_title":"AI Integrity: Defending Against Backdoors and Secret Loyalties","ref_index":20,"is_internal_anchor":true},{"citing_arxiv_id":"2606.21972","citing_title":"Human vs Machine Mathematical Difficulty on Project Euler: An Experimental Analysis","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2606.21627","citing_title":"Counsel: A Meta-Evaluation Dataset for Agentic Tasks","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2606.19613","citing_title":"StaminaBench: Stress-Testing Coding Agents over 100 Interaction Turns","ref_index":26,"is_internal_anchor":true},{"citing_arxiv_id":"2606.17887","citing_title":"AI Adoption Across a Multinational Workforce: Sociotechnical Conditions for GenAI Acceptance in Human Resources","ref_index":79,"is_internal_anchor":true},{"citing_arxiv_id":"2606.20683","citing_title":"From Question Answering to Task Completion: A Survey on Agent System and Harness Design","ref_index":95,"is_internal_anchor":true},{"citing_arxiv_id":"2606.10711","citing_title":"The Agentic Web Requires New Normative Infrastructure","ref_index":260,"is_internal_anchor":true},{"citing_arxiv_id":"2606.10127","citing_title":"Data-Driven Automation","ref_index":34,"is_internal_anchor":true},{"citing_arxiv_id":"2606.08867","citing_title":"Building Customer Support AI Agents at 100M-User Scale: An Evaluation-Driven Framework","ref_index":25,"is_internal_anchor":true},{"citing_arxiv_id":"2606.08529","citing_title":"Scaffold Effects on GAIA: A Controlled Comparison","ref_index":4,"is_internal_anchor":true},{"citing_arxiv_id":"2606.08340","citing_title":"Benchmarking Open-Ended Multi-Agent Coordination in Language Agents","ref_index":31,"is_internal_anchor":true},{"citing_arxiv_id":"2606.07157","citing_title":"Think Fast: Estimating No-CoT Task-Completion Time Horizons of Frontier AI Models","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2606.04455","citing_title":"The Meta-Agent Challenge: Are Current Agents Capable of Autonomous Agent Development?","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2606.03461","citing_title":"What Makes Interaction Trajectories Effective for Training Terminal Agents?","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2606.27416","citing_title":"Glite ARF: Verifier-Driven Research with Parallel LLM Coding Agents","ref_index":50,"is_internal_anchor":true},{"citing_arxiv_id":"2606.31916","citing_title":"Theory of Mind and Persuasion Beyond Conversation: Assessing the Capacity of LLMs to Induce Belief States via Planning and Action","ref_index":62,"is_internal_anchor":true},{"citing_arxiv_id":"2605.15164","citing_title":"Position: Behavioural Assurance Cannot Verify the Safety Claims Governance Now Demands","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14675","citing_title":"Agentic AI in Industry: Adoption Level and Deployment Barriers","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.27485","citing_title":"Automating Formal Verification with Agent-Guided Tree Search","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2606.07157","citing_title":"Think Fast: Estimating No-CoT Task-Completion Time Horizons of Frontier AI Models","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2605.30169","citing_title":"Dissociative Identity: Language Model Agents Lack Grounding for Reputation Mechanisms","ref_index":73,"is_internal_anchor":true},{"citing_arxiv_id":"2606.10711","citing_title":"The Agentic Web Requires New Normative Infrastructure","ref_index":119,"is_internal_anchor":true},{"citing_arxiv_id":"2606.11926","citing_title":"Toward Generalist Autonomous Research via Hypothesis-Tree Refinement","ref_index":129,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN","json":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN.json","graph_json":"https://pith.science/api/pith-number/LVIF2CDCVEMAT56A6MVCSEPCWN/graph.json","events_json":"https://pith.science/api/pith-number/LVIF2CDCVEMAT56A6MVCSEPCWN/events.json","paper":"https://pith.science/paper/LVIF2CDC"},"agent_actions":{"view_html":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN","download_json":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN.json","view_paper":"https://pith.science/paper/LVIF2CDC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2503.14499&json=true","fetch_graph":"https://pith.science/api/pith-number/LVIF2CDCVEMAT56A6MVCSEPCWN/graph.json","fetch_events":"https://pith.science/api/pith-number/LVIF2CDCVEMAT56A6MVCSEPCWN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN/action/storage_attestation","attest_author":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN/action/author_attestation","sign_citation":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN/action/citation_signature","submit_replication":"https://pith.science/pith/LVIF2CDCVEMAT56A6MVCSEPCWN/action/replication_record"}},"created_at":"2026-07-14T00:18:18.548681+00:00","updated_at":"2026-07-14T00:18:18.548681+00:00"}