{"work":{"id":"7db23490-3dd0-4cfa-a75c-5537ee36dcc2","openalex_id":null,"doi":null,"arxiv_id":"2504.01848","raw_key":null,"title":"PaperBench: Evaluating AI's Ability to Replicate AI Research","authors":null,"authors_text":"Giulio Starace, Oliver Jaffe, Dane Sherburn, James Aung, Jun Shern Chan, Leon Maksin","year":2025,"venue":"cs.AI","abstract":"We introduce PaperBench, a benchmark evaluating the ability of AI agents to replicate state-of-the-art AI research. Agents must replicate 20 ICML 2024 Spotlight and Oral papers from scratch, including understanding paper contributions, developing a codebase, and successfully executing experiments. For objective evaluation, we develop rubrics that hierarchically decompose each replication task into smaller sub-tasks with clear grading criteria. In total, PaperBench contains 8,316 individually gradable tasks. Rubrics are co-developed with the author(s) of each ICML paper for accuracy and realism. To enable scalable evaluation, we also develop an LLM-based judge to automatically grade replication attempts against rubrics, and assess our judge's performance by creating a separate benchmark for judges. We evaluate several frontier models on PaperBench, finding that the best-performing tested agent, Claude 3.5 Sonnet (New) with open-source scaffolding, achieves an average replication score of 21.0%. Finally, we recruit top ML PhDs to attempt a subset of PaperBench, finding that models do not yet outperform the human baseline. We open-source our code (https://github.com/openai/preparedness) to facilitate future research in understanding the AI engineering capabilities of AI agents.","external_url":"https://arxiv.org/abs/2504.01848","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-05-25T05:45:23.255059+00:00","pith_arxiv_id":"2504.01848","created_at":"2026-05-09T01:14:32.607740+00:00","updated_at":"2026-05-25T05:45:23.255059+00:00","title_quality_ok":true,"display_title":"PaperBench: Evaluating AI's Ability to Replicate AI Research","render_title":"PaperBench: Evaluating AI's Ability to Replicate AI Research"},"hub":{"state":{"work_id":"7db23490-3dd0-4cfa-a75c-5537ee36dcc2","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":34,"external_cited_by_count":null,"distinct_field_count":12,"first_pith_cited_at":"2025-06-27T19:41:41+00:00","last_pith_cited_at":"2026-05-22T03:40:30+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-05-27T21:37:57.707538+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":6},{"context_role":"dataset","n":2},{"context_role":"method","n":1}],"polarity_counts":[{"context_polarity":"background","n":6},{"context_polarity":"use_dataset","n":2},{"context_polarity":"use_method","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}