{"as_of":"2026-08-16T06:08:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c3e513da4dd94018062eaf2a0a3384c73a5f98b2e018b611ee8354574069fcf7","coverage":[{"denominator":64,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":64,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T14:33:13.231482Z","state":"measured"},{"denominator":126,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":126,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":62,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":62,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T05:44:03.801916Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"pith","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":2,"observed_at":"2026-08-05T02:28:24.338817Z","source":"pith"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2412.04984","last_updated":"2025-01-14T20:16:01Z","snapshot_observed_at":"2026-08-15T04:02:05.429942Z","submitted_at":"2024-12-06T12:09:50Z","title":"Frontier Models are Capable of In-context Scheming","version":2},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-05-16T14:22:01.616448Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2412.04984"},"observation_digest":"sha256:3738b5416d27a5529726d699798f1d58cc4972898831e1f6ad91fbc94b66c114","observation_id":"4b89afad-6c15-4cd6-a734-fdf2480c86bc","resolution":{"observed_at":"2026-05-16T14:22:01.681340Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2501.14249","last_updated":"2026-02-20T04:23:01Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-01-24T05:27:46Z","title":"Humanity's Last Exam","version":10},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-05-10T18:40:50.139345Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2501.14249"},"observation_digest":"sha256:d6fef5d853a3bda0ee0881701784dffff99232ca3f81b42fb5f4735195d07335","observation_id":"d9fa4ac5-63e7-45de-986b-c9ba862334fb","resolution":{"observed_at":"2026-05-10T18:40:50.326504Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-09T14:48:35.872296Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.01635","last_updated":"2025-02-03T18:59:13Z","snapshot_observed_at":"2026-08-15T06:24:43.729406Z","submitted_at":"2025-02-03T18:59:13Z","title":"The AI Agent Index","version":1},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-08-09T14:48:35.872296Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2502.01635"},"observation_digest":"sha256:533822e44d4e1db6cb0c3280de09d9c07d791639f41ec8be9c96f0d57e6ab336","observation_id":"7dddfdc1-9de0-4ea1-b0b6-ecb02f46e6d0","resolution":{"observed_at":"2026-08-09T14:48:35.872296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2502.10517","last_updated":"2025-02-14T19:30:53Z","snapshot_observed_at":"2026-08-14T07:16:55.895519Z","submitted_at":"2025-02-14T19:30:53Z","title":"KernelBench: Can LLMs Write Efficient GPU Kernels?","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-05-15T16:55:01.976356Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2502.10517"},"observation_digest":"sha256:179cf4ce037cd0dc855daa5bb2d8cddd57beeabc26b13a36dfb0556ed82e07b6","observation_id":"0db8ffb1-e533-489d-b388-a0a9ea0c6fec","resolution":{"observed_at":"2026-05-15T16:55:02.090533Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-16T05:44:03.801916Z","title":"and Ericheva, E., 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.19912","last_updated":"2025-04-28T15:41:28Z","snapshot_observed_at":"2026-08-16T05:37:46.940986Z","submitted_at":"2025-04-28T15:41:28Z","title":"Can AI Agents Design and Implement Drug Discovery Pipelines?","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-16T05:44:03.801916Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2504.19912"},"observation_digest":"sha256:67e498ab5af32e935626e007b4a323a39801e4e833592c65fb82c321b0ed49d2","observation_id":"7f9a1d21-dc25-471a-8e49-39fd2441209b","resolution":{"observed_at":"2026-08-16T05:44:03.801916Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-15T23:27:30.618243Z","title":"RE - Bench : Evaluating frontier AI R & D capabilities of language model agents against human experts, November 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.04592","last_updated":"2025-05-07T17:35:36Z","snapshot_observed_at":"2026-08-15T23:22:24.607476Z","submitted_at":"2025-05-07T17:35:36Z","title":"AI Governance to Avoid Extinction: The Strategic Landscape and Actionable Research Questions","version":1},"reference_index":195,"source":"arxiv_source","source_observed_at":"2026-08-15T23:27:30.618243Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2505.04592"},"observation_digest":"sha256:faab58e6d2a1918208e684066e0939361dff58e46f4f1b0727d88a7c7ff41727","observation_id":"19dcc8fb-763b-47e7-8ec9-771e5de241fe","resolution":{"observed_at":"2026-08-15T23:27:30.618243Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-15T22:50:49.704789Z","title":"unspecialized humans","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.06108","last_updated":"2025-05-21T20:34:27Z","snapshot_observed_at":"2026-08-15T22:45:51.401100Z","submitted_at":"2025-05-09T15:05:57Z","title":"LLMs Outperform Experts on Challenging Biology Benchmarks","version":3},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T22:50:49.704789Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2505.06108"},"observation_digest":"sha256:3fc2f93c196f5919eb36bc8ab84b960d0c072d6054abe80009989fcdacdad9dd","observation_id":"35d19ee8-5d53-4ede-b2c2-bd9c4fc69316","resolution":{"observed_at":"2026-08-15T22:50:49.704789Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-15T20:18:59.964481Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.13291","last_updated":"2025-05-19T16:11:23Z","snapshot_observed_at":"2026-08-15T20:13:53.875819Z","submitted_at":"2025-05-19T16:11:23Z","title":"TimeSeriesGym: A Scalable Benchmark for (Time Series) Machine Learning Engineering Agents","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T20:18:59.964481Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2505.13291"},"observation_digest":"sha256:f52a3e1164ba3651f71c2f74d859db4f98e09ca76712cdba9215c01746b51bf1","observation_id":"79ec15b4-8781-468b-8d3e-beeea998a9e1","resolution":{"observed_at":"2026-08-15T20:18:59.964481Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-07T12:18:55.881071Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24876","last_updated":"2026-05-24T03:59:43Z","snapshot_observed_at":"2026-08-15T02:52:54.149121Z","submitted_at":"2025-05-30T17:59:53Z","title":"Agent-X: Evaluating Deep Multimodal Reasoning in Vision-Centric Agentic Tasks","version":2},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-07T12:18:55.881071Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2505.24876"},"observation_digest":"sha256:4cc5d8bc5a3a3f1262d7fbd317103c3ffe5de0129c1ade473e1b4c9e482f8786","observation_id":"e5b67da1-0357-4675-8eb8-5547e5883c99","resolution":{"observed_at":"2026-08-07T12:18:55.881071Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-07T10:51:58.052862Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.04098","last_updated":"2025-06-10T13:14:14Z","snapshot_observed_at":"2026-08-15T10:08:43.288385Z","submitted_at":"2025-06-04T15:55:27Z","title":"TextAtari: 100K Frames Game Playing with Language Agents","version":2},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-07T10:51:58.052862Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2506.04098"},"observation_digest":"sha256:ea0f53e8097aff79c738d2abf5466eaf253cb4a5de07bda3d7cb9acb7385a98c","observation_id":"603c9afb-9026-40fe-9f5a-dc6febb693fb","resolution":{"observed_at":"2026-08-07T10:51:58.052862Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T23:26:56.952422Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18096","last_updated":"2025-09-03T15:32:23Z","snapshot_observed_at":"2026-08-15T20:12:18.032530Z","submitted_at":"2025-06-22T16:52:48Z","title":"Deep Research Agents: A Systematic Examination And Roadmap","version":2},"reference_index":118,"source":"pdf_text","source_observed_at":"2026-08-06T23:26:56.952422Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2506.18096"},"observation_digest":"sha256:f9dd06ecc215d37107a12abb1caec4b9da552f217b8e038cfef346afc44f71a7","observation_id":"f650702d-9842-4d40-99ca-3e58f8cf9c10","resolution":{"observed_at":"2026-08-06T23:26:56.952422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T20:24:27.421486Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.02825","last_updated":"2025-08-07T06:58:08Z","snapshot_observed_at":"2026-08-15T11:03:19.944695Z","submitted_at":"2025-07-03T17:35:31Z","title":"Establishing Best Practices for Building Rigorous Agentic Benchmarks","version":5},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T20:24:27.421486Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.02825"},"observation_digest":"sha256:c55f1d4ffed103fe51932d2596c5d95bbbddec9c9476ff4cacf345aba7dce311","observation_id":"36100b82-0a6a-453d-8fd8-7252d4bb2915","resolution":{"observed_at":"2026-08-06T20:24:27.421486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T19:39:59.279843Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.06260","last_updated":"2025-07-07T13:33:35Z","snapshot_observed_at":"2026-08-14T18:52:26.445135Z","submitted_at":"2025-07-07T13:33:35Z","title":"Evaluating the Critical Risks of Amazon's Nova Premier under the Frontier Model Safety Framework","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T19:39:59.279843Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.06260"},"observation_digest":"sha256:be7d02542125b24cba13ae833c8178249737998d7875623d1c8d38a19047086e","observation_id":"b3799e82-49f1-4e93-b917-0b42741574ce","resolution":{"observed_at":"2026-08-06T19:39:59.279843Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2507.06261","last_updated":"2025-12-19T14:25:46Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-07-07T17:36:04Z","title":"Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities","version":6},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-05-19T05:48:02.828938Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.06261"},"observation_digest":"sha256:70a5940926972b694076bc62ad7dfaba33d19c473d0377cf6343fde773e627da","observation_id":"a58694cb-4e26-4f00-876c-ee10cd3eb27b","resolution":{"observed_at":"2026-05-19T05:52:07.806483Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T19:00:08.585286Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.08038","last_updated":"2026-06-01T08:32:26Z","snapshot_observed_at":"2026-08-12T14:12:59.779539Z","submitted_at":"2025-07-09T12:07:38Z","title":"AblationBench: Evaluating Automated Planning of Ablations in Empirical AI Research","version":3},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-06T19:00:08.585286Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.08038"},"observation_digest":"sha256:a01f14d84bc565edbfbc6fe054b7c5e2cf7be00afee00f657f9e1c5b3924b546","observation_id":"9e58febf-03fe-4d5f-8648-635514eec14a","resolution":{"observed_at":"2026-08-06T19:00:08.585286Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T18:24:26.153839Z","title":"Non-Linear Story Understanding","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.08350","last_updated":"2025-07-11T06:53:46Z","snapshot_observed_at":"2026-08-12T16:50:18.403288Z","submitted_at":"2025-07-11T06:53:46Z","title":"Exploring Design of Multi-Agent LLM Dialogues for Research Ideation","version":1},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-06T18:24:26.153839Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.08350"},"observation_digest":"sha256:e674d241e47498c2cdb2b56ec9f01f1ab62a2b97f1d8430ff5312bd984072d71","observation_id":"09cd1878-e82b-4063-bde5-4ff58152aeb2","resolution":{"observed_at":"2026-08-06T18:24:26.153839Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T18:12:29.939412Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.09089","last_updated":"2025-07-25T00:43:07Z","snapshot_observed_at":"2026-08-15T10:22:55.094062Z","submitted_at":"2025-07-12T00:16:33Z","title":"Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T18:12:29.939412Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.09089"},"observation_digest":"sha256:1204087be24a07a17285fe13174a21a21419c9548c76adc9935a5d93beaebeed","observation_id":"203e8758-23fb-496e-bb2c-9753f4b9bdb4","resolution":{"observed_at":"2026-08-06T18:12:29.939412Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2507.11473","last_updated":"2025-12-07T02:14:12Z","snapshot_observed_at":"2026-08-10T09:38:50.886870Z","submitted_at":"2025-07-15T16:43:41Z","title":"Chain of Thought Monitorability: A New and Fragile Opportunity for AI Safety","version":2},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-05-20T14:19:44.695462Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2507.11473"},"observation_digest":"sha256:20b02c9c35ab60cbfd4bec4308c459039c348b94ec208008735c8aea2ae5aae3","observation_id":"7eca4a44-634e-4288-828e-ad2fcd68d077","resolution":{"observed_at":"2026-05-20T14:19:44.776446Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2510.12826","last_updated":"2026-04-25T22:54:37Z","snapshot_observed_at":"2026-08-15T03:26:09.122820Z","submitted_at":"2025-10-11T04:42:29Z","title":"Scheming Ability in LLM-to-LLM Strategic Interactions","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-18T07:50:30.597108Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2510.12826"},"observation_digest":"sha256:ec10cc462ce37988575f0aecae656a666c36de3340bcaa0193187d03838d1b56","observation_id":"f89e799f-9104-420c-9c93-7bfaa6b22443","resolution":{"observed_at":"2026-05-18T07:51:03.835431Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2602.09514","last_updated":"2026-05-09T05:05:58Z","snapshot_observed_at":"2026-08-11T14:00:59.059145Z","submitted_at":"2026-02-10T08:12:23Z","title":"EcoGym: Evaluating LLMs for Long-Horizon Plan-and-Execute in Interactive Economies","version":3},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-05-16T05:53:29.860037Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2602.09514"},"observation_digest":"sha256:5a17c9db12b95f67f6a1a01b75c15280331bca1014b2010d0f637608b7786238","observation_id":"76b13409-8d2b-4366-afea-6a06c93a1a20","resolution":{"observed_at":"2026-05-16T05:57:24.639150Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-02T20:36:06.369914Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2602.22971","last_updated":"2026-05-29T07:44:51Z","snapshot_observed_at":"2026-08-15T17:16:03.296022Z","submitted_at":"2026-02-26T13:08:56Z","title":"SPM-Bench: Benchmarking Large Language Models for Scanning Probe Microscopy","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-02T20:36:06.369914Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2602.22971"},"observation_digest":"sha256:3f70c4cc2a96fc10b42f5f195d0b64ba71d7893abe453de433f9a9b7b03d46a6","observation_id":"9126949c-9665-45a0-b476-7436e91deb07","resolution":{"observed_at":"2026-08-02T20:36:06.369914Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2604.12290","last_updated":"2026-04-27T16:57:20Z","snapshot_observed_at":"2026-08-14T21:22:33.612209Z","submitted_at":"2026-04-14T05:02:06Z","title":"Frontier-Eng: Benchmarking Self-Evolving Agents on Real-World Engineering Tasks with Generative Optimization","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-05-10T16:17:32.290531Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2604.12290"},"observation_digest":"sha256:3fed2cece00446916607a3b230ab08a356c0ffc19e2548a67864375b45397b2b","observation_id":"b030444d-0020-438e-a874-0edb4351c374","resolution":{"observed_at":"2026-05-11T09:05:57.739286Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2604.14116","last_updated":"2026-04-22T07:33:13Z","snapshot_observed_at":"2026-08-12T21:10:39.427401Z","submitted_at":"2026-04-15T17:38:06Z","title":"TREX: Automating LLM Fine-tuning via Agent-Driven Tree-based Exploration","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-05-10T12:34:29.808503Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2604.14116"},"observation_digest":"sha256:252e6a1b6c088f5670460e5003d594816edfd7642fac8e8fe96ef0e6ff8b4895","observation_id":"8c7a22b9-3c70-4efa-97fd-886a14846ce8","resolution":{"observed_at":"2026-05-11T11:46:34.653244Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2604.24966","last_updated":"2026-04-27T20:07:09Z","snapshot_observed_at":"2026-08-13T15:26:57.998704Z","submitted_at":"2026-04-27T20:07:09Z","title":"Risk Reporting for Developers' Internal AI Model Use","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-05-07T17:47:21.321820Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2604.24966"},"observation_digest":"sha256:b02f5a2a36cd6cf71310b477116367f5ab735f8e8eedfac20ae283ee24b21360","observation_id":"c7d436cc-01f1-4907-92f6-2883a4edf091","resolution":{"observed_at":"2026-05-11T23:16:16.444053Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.02050","last_updated":"2026-07-26T23:34:38Z","snapshot_observed_at":"2026-08-02T15:01:22.490123Z","submitted_at":"2026-05-03T20:37:48Z","title":"Principles and Guidelines for Randomized Controlled Trials in AI Evaluation","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-05-08T18:55:02.645026Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.02050"},"observation_digest":"sha256:affd2240743be3dd4ca4c214769d8d3df0c43cd6fc64565ba51efe6697ada9d0","observation_id":"43f82950-7e26-4aa2-994c-4d56fa16925b","resolution":{"observed_at":"2026-05-09T06:05:36.486105Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.13950","last_updated":"2026-05-13T18:00:00Z","snapshot_observed_at":"2026-08-13T15:32:30.494754Z","submitted_at":"2026-05-13T18:00:00Z","title":"Collider-Bench: Benchmarking AI Agents with Particle Physics Analysis Reproduction","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-05-15T06:04:03.605898Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.13950"},"observation_digest":"sha256:022fde0e5ee8ef3f50374a17436b472f450210b5abd228b533246c6e0dba5318","observation_id":"b4e7854c-6681-4446-ae70-14c4cba12e9a","resolution":{"observed_at":"2026-05-15T06:05:06.762989Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.14445","last_updated":"2026-05-14T06:39:42Z","snapshot_observed_at":"2026-08-01T15:27:40.908509Z","submitted_at":"2026-05-14T06:39:42Z","title":"FrontierSmith: Synthesizing Open-Ended Coding Problems at Scale","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-05-15T02:02:25.597640Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.14445"},"observation_digest":"sha256:9b16db98067f435933df1fb0898e0c7f46b8c7317cba50cc71838fd7f59dd870","observation_id":"002c6092-2b3e-4304-a6b9-38cc4ca0c284","resolution":{"observed_at":"2026-05-15T02:03:29.062735Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.17373","last_updated":"2026-05-29T04:24:32Z","snapshot_observed_at":"2026-08-15T09:26:02.151681Z","submitted_at":"2026-05-17T10:30:38Z","title":"FML-bench: A Controlled Study of AI Research Agent Strategies from the Perspective of Search Dynamics","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-05-20T14:25:15.565386Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.17373"},"observation_digest":"sha256:5abaaf574e15262a7fcd4e8b5f9d0d121f7dfaf166347f8d19223792b20492b6","observation_id":"f16d3fd5-9469-4b23-8d75-671c0be93582","resolution":{"observed_at":"2026-05-20T14:28:21.448976Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.17373","last_updated":"2026-05-29T04:24:32Z","snapshot_observed_at":"2026-08-15T09:26:02.151681Z","submitted_at":"2026-05-17T10:30:38Z","title":"FML-bench: A Controlled Study of AI Research Agent Strategies from the Perspective of Search Dynamics","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-06-30T19:00:30.961402Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.17373"},"observation_digest":"sha256:bb06bfbf23d4f33821b1e374312427ec14f42c86a7c8ac1c541001a9892c3ffc","observation_id":"98f353b7-e769-4be9-910b-36a33c63fd71","resolution":{"observed_at":"2026-06-30T19:05:00.924661Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.18661","last_updated":"2026-07-20T17:24:03Z","snapshot_observed_at":"2026-08-14T09:37:41.438142Z","submitted_at":"2026-05-18T17:08:26Z","title":"AI for Auto-Research: Roadmap & User Guide","version":1},"reference_index":222,"source":"pdf_text","source_observed_at":"2026-05-20T10:30:50.256635Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.18661"},"observation_digest":"sha256:4f6e8b579369bccbd86ba90bde1d09774a103bd6f388e213f6efa72bf90c373c","observation_id":"f9630578-942a-41d2-9260-a9449fdd468e","resolution":{"observed_at":"2026-05-20T10:33:12.557130Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-02T13:43:49.951321Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2605.18661","last_updated":"2026-07-20T17:24:03Z","snapshot_observed_at":"2026-08-14T09:37:41.438142Z","submitted_at":"2026-05-18T17:08:26Z","title":"AI for Auto-Research: Roadmap & User Guide","version":2},"reference_index":221,"source":"pdf_text","source_observed_at":"2026-08-02T13:43:49.951321Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.18661"},"observation_digest":"sha256:c1a0816d13b2963910603eafb33ba3e2166c69bf5fc9293fbcae77f4530df5cd","observation_id":"5328db6c-78ad-45aa-86de-e3618d706e3f","resolution":{"observed_at":"2026-08-02T13:43:49.951321Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.19156","last_updated":"2026-05-18T22:20:33Z","snapshot_observed_at":"2026-08-15T00:50:05.932495Z","submitted_at":"2026-05-18T22:20:33Z","title":"How Far Are We From True Auto-Research?","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-05-20T09:56:16.160551Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.19156"},"observation_digest":"sha256:c292869ec620f3fb728b5d0549938dc8d0f0a4ed9d65737d96db7d39f50e58d4","observation_id":"321ddc94-9675-4172-88d5-3f37ac4c7741","resolution":{"observed_at":"2026-05-20T09:58:11.242622Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.20744","last_updated":"2026-05-20T05:46:52Z","snapshot_observed_at":"2026-08-16T05:57:15.518994Z","submitted_at":"2026-05-20T05:46:52Z","title":"Hack-Verifiable Environments: Towards Evaluating Reward Hacking at Scale","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-05-21T06:56:27.532299Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.20744"},"observation_digest":"sha256:f9965a062ac02122cb13acfbe87ba21649a585b83028791bf9c8c8e7ddc54212","observation_id":"95e6e6dd-7a45-4c86-a98d-45990b1871f9","resolution":{"observed_at":"2026-05-21T06:59:45.559572Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2605.27492","last_updated":"2026-05-26T16:28:10Z","snapshot_observed_at":"2026-08-07T15:27:01.984383Z","submitted_at":"2026-05-26T16:28:10Z","title":"Benchmarks are Not Enough: RAMP for Runtime Assessing of Agentic Models in Production Systems","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-06-29T15:32:21.737028Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2605.27492"},"observation_digest":"sha256:69ecf41710f212c4b2566781d2c327be331b0d57c9f36fb1b6ba700bda86b44c","observation_id":"23a23acf-0525-40c6-9917-a28501eda992","resolution":{"observed_at":"2026-06-29T15:33:32.798892Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.04455","last_updated":"2026-06-03T04:58:17Z","snapshot_observed_at":"2026-08-06T11:33:07.154821Z","submitted_at":"2026-06-03T04:58:17Z","title":"The Meta-Agent Challenge: Are Current Agents Capable of Autonomous Agent Development?","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-06-28T06:29:42.665765Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.04455"},"observation_digest":"sha256:b6c91716920eb5f31b9611a839613c1e0dc7d6410423031ad31b06c785500746","observation_id":"abf9f5f6-6c96-483c-8388-2bd6d7e356b7","resolution":{"observed_at":"2026-07-02T07:56:47.690735Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.05080","last_updated":"2026-06-03T16:36:54Z","snapshot_observed_at":"2026-08-15T05:12:19.806726Z","submitted_at":"2026-06-03T16:36:54Z","title":"AutoLab: Can Frontier Models Solve Long-Horizon Auto Research and Engineering Tasks?","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-06-28T06:37:03.670571Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.05080"},"observation_digest":"sha256:ecddfdc045d4d98fce2d9644e48aaa9d8962a0eaa314cdf5d4cec1f8eaf702cf","observation_id":"52945ae3-1b6f-4313-b036-adb3f275750d","resolution":{"observed_at":"2026-07-02T07:46:46.619488Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.09550","last_updated":"2026-06-08T14:29:40Z","snapshot_observed_at":"2026-08-13T02:09:33.292917Z","submitted_at":"2026-06-08T14:29:40Z","title":"InquiTree: Evaluating AI Agents in the Scientific Inquiry Loop with Paper-Derived Research Trees","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-06-27T14:06:59.471772Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.09550"},"observation_digest":"sha256:fd2804742000214a1bccdb55d7587b5a5b34106dffa0f42200dc8ff866ea8fcc","observation_id":"9668af3d-f1e5-45f1-98eb-3e0245013815","resolution":{"observed_at":"2026-07-03T04:07:37.255049Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.11926","last_updated":"2026-06-10T10:57:05Z","snapshot_observed_at":"2026-08-09T20:13:04.512653Z","submitted_at":"2026-06-10T10:57:05Z","title":"Toward Generalist Autonomous Research via Hypothesis-Tree Refinement","version":1},"reference_index":160,"source":"arxiv_source","source_observed_at":"2026-06-27T09:34:41.800309Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.11926"},"observation_digest":"sha256:8b546ed539f4bf930dd2006d3150c623044a060e7ddd2a1f927c6a724288e83e","observation_id":"efde55ab-60f9-4aae-8e1d-c230c3c7495a","resolution":{"observed_at":"2026-06-27T09:40:47.033546Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.17799","last_updated":"2026-07-18T22:14:13Z","snapshot_observed_at":"2026-08-02T11:05:59.500924Z","submitted_at":"2026-06-16T11:21:01Z","title":"Position: Coding Benchmarks Are Misaligned with Agentic Software Engineering","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-06-26T23:48:58.497927Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.17799"},"observation_digest":"sha256:027a1ea90308be9b7462038c42cb5b12f0afb8c2d427e0592606c42118b08e9c","observation_id":"4cc5987e-a720-497d-9c48-74c0d0c44ba6","resolution":{"observed_at":"2026-07-03T22:08:58.899735Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-02T11:06:03.847203Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2606.17799","last_updated":"2026-07-18T22:14:13Z","snapshot_observed_at":"2026-08-02T11:05:59.500924Z","submitted_at":"2026-06-16T11:21:01Z","title":"Position: Coding Benchmarks Are Misaligned with Agentic Software Engineering","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-02T11:06:03.847203Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.17799"},"observation_digest":"sha256:6443cfded5d10bd389dd119423b42145a693f9226296a276aad2b57288a71cc0","observation_id":"5d39a32f-a09d-4ee2-a9dc-45b587457b01","resolution":{"observed_at":"2026-08-02T11:06:03.847203Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.21891","last_updated":"2026-06-20T05:47:37Z","snapshot_observed_at":"2026-08-15T05:35:21.274747Z","submitted_at":"2026-06-20T05:47:37Z","title":"Learning the ARTS of Search for Automated Discovery","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-06-26T12:04:13.117307Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.21891"},"observation_digest":"sha256:23765d668ec084525433b209fc3ac09802b3bd4d80aa1b005b0fcf85c094b069","observation_id":"4806bec1-2f3f-447a-b62b-400a84e06fa3","resolution":{"observed_at":"2026-07-04T08:09:41.833473Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.22866","last_updated":"2026-06-22T05:24:10Z","snapshot_observed_at":"2026-08-16T05:49:30.674400Z","submitted_at":"2026-06-22T05:24:10Z","title":"Discovering Crystal Structure Prediction Algorithms with an AI Co-Scientist","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-06-26T09:03:00.675456Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.22866"},"observation_digest":"sha256:3ce0046e3bc3aa47c87ad5c71b4cab337a667985c19dde234b94a7903eca7b9d","observation_id":"1ac7c5ff-0cf2-4e78-b901-f62eaf2d6854","resolution":{"observed_at":"2026-07-04T10:19:46.987416Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2606.30182","last_updated":"2026-07-17T10:56:56Z","snapshot_observed_at":"2026-08-08T10:47:58.204777Z","submitted_at":"2026-06-29T11:57:32Z","title":"MirrorCode: AI can rebuild entire programs from behavior alone","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-06-30T06:35:32.667865Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.30182"},"observation_digest":"sha256:08c084cf78f6bb508f4f8a3e122f74c1b19abdc0809e93ac5da118cce29dd9a3","observation_id":"a0aaa883-453e-4868-ac34-d39a298bd0e0","resolution":{"observed_at":"2026-06-30T06:44:19.703971Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-02T09:35:02.886229Z","title":"RE-bench: Evaluating frontier AI R&D capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.30182","last_updated":"2026-07-17T10:56:56Z","snapshot_observed_at":"2026-08-08T10:47:58.204777Z","submitted_at":"2026-06-29T11:57:32Z","title":"MirrorCode: AI can rebuild entire programs from behavior alone","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-02T09:35:02.886229Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2606.30182"},"observation_digest":"sha256:347dfdded85dc3447406893c417426bcd999ad0e1e2c16ebaa3fdd698a46ca7f","observation_id":"c974db13-d3e9-4d21-8d11-6794418d9f38","resolution":{"observed_at":"2026-08-02T09:35:02.886229Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2607.00913","last_updated":"2026-07-01T13:18:21Z","snapshot_observed_at":"2026-08-12T14:34:19.802783Z","submitted_at":"2026-07-01T13:18:21Z","title":"Two AI Metrics Diverged: Will it Make All the Difference?","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-07-02T12:29:24.439779Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.00913"},"observation_digest":"sha256:bde5d7697fc0c6ea7d6b219777d7c6e96894f4ff5d065dd1e76f0be2c82a9f8f","observation_id":"9c361421-912f-4c8e-8c9b-46ec8d5cb72a","resolution":{"observed_at":"2026-07-02T12:36:56.156858Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":"2411.15114","doi":"10.48550/arxiv.2411.15114","metadata_source":"pith","pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts","venue":"cs.LG","work_id":"f6a7efa2-8f95-4bbf-9931-611f36393c20","year":2024},"citing_paper":{"arxiv_id":"2607.06411","last_updated":"2026-07-19T18:49:59Z","snapshot_observed_at":"2026-08-12T02:44:21.439415Z","submitted_at":"2026-07-07T15:41:22Z","title":"RuBench: A Repository-Level Agentic Coding Benchmark with Natively Authored Russian Task Specifications","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-07-08T06:55:40.830922Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.06411"},"observation_digest":"sha256:e589c9bbb1043d66bd9ae9238a167bad5ace77d4955be321afa40689482b71fd","observation_id":"4031574a-1bc3-4e7d-84a4-046c5a8f602e","resolution":{"observed_at":"2026-07-08T07:04:44.520871Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-02T08:21:50.147998Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.06411","last_updated":"2026-07-19T18:49:59Z","snapshot_observed_at":"2026-08-12T02:44:21.439415Z","submitted_at":"2026-07-07T15:41:22Z","title":"RuBench: A Repository-Level Agentic Coding Benchmark with Natively Authored Russian Task Specifications","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-02T08:21:50.147998Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.06411"},"observation_digest":"sha256:562d408458868d27510957265461e65f5643ce2d41d8345d1a4fa17bfd5e8119","observation_id":"58fcc192-7e31-4da4-944f-af8b18859fb3","resolution":{"observed_at":"2026-08-02T08:21:50.147998Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-07-13T05:28:45.311405Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.08964","last_updated":"2026-07-13T06:00:47Z","snapshot_observed_at":"2026-08-16T01:34:23.512255Z","submitted_at":"2026-07-09T21:56:37Z","title":"Long-Horizon-Terminal-Bench: Testing the Limits of Agents on Long-Horizon Terminal Tasks with Dense Reward-Based Grading","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-07-13T05:28:45.311405Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.08964"},"observation_digest":"sha256:cceb873daa8cdc955fc0d26aed7976928d3b69e65f01d19bef9108634a2932ce","observation_id":"1296820b-a97b-4a1e-ab5e-dfdc4571568b","resolution":{"observed_at":"2026-07-13T05:28:45.311405Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-07-14T15:24:58.589243Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.08964","last_updated":"2026-07-13T06:00:47Z","snapshot_observed_at":"2026-08-16T01:34:23.512255Z","submitted_at":"2026-07-09T21:56:37Z","title":"Long-Horizon-Terminal-Bench: Testing the Limits of Agents on Long-Horizon Terminal Tasks with Dense Reward-Based Grading","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-07-14T15:24:58.589243Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.08964"},"observation_digest":"sha256:c70563468fd18bdcb420138639509f18e1250c2880c9e4d22cd79e90f2e193f1","observation_id":"c4e9e2a6-d05e-4c08-b8cf-89f4e4b97842","resolution":{"observed_at":"2026-07-14T15:24:58.589243Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-07-14T10:46:01.272433Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10569","last_updated":"2026-07-12T04:52:08Z","snapshot_observed_at":"2026-08-14T09:00:20.788452Z","submitted_at":"2026-07-12T04:52:08Z","title":"When Does Restricting a Coding Agent to execute_code Help? A Regime $\\times$ Agent-Design Ablation","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-07-14T10:46:01.272433Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.10569"},"observation_digest":"sha256:0f4be3b2c32cec459ba7e8cfa0f4d0ed4cb3c9d3d9a7fb9caf70f6d1b5b08296","observation_id":"4300af11-4056-4bf9-af7a-3b20f2f358d5","resolution":{"observed_at":"2026-07-14T10:46:01.272433Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-01T12:50:00.490138Z","title":"RE-bench: Evaluating frontier AI R&D capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.19321","last_updated":"2026-07-29T16:23:28Z","snapshot_observed_at":"2026-08-12T18:30:22.824623Z","submitted_at":"2026-07-21T17:41:12Z","title":"ResearchArena: Evaluating Sabotage and Monitoring in Automated AI R&D","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-01T12:50:00.490138Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.19321"},"observation_digest":"sha256:8cc8e5151bf05d09b15ecd01c77b7d76427c992e83630d79418dcadda89e7e3c","observation_id":"45bc81b3-20ed-4ca4-a3a7-bf39ecc86496","resolution":{"observed_at":"2026-08-01T12:50:00.490138Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-07-31T09:14:06.207902Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.24647","last_updated":"2026-07-27T16:46:33Z","snapshot_observed_at":"2026-08-08T15:36:01.479405Z","submitted_at":"2026-07-27T16:46:33Z","title":"Efficiency Matters in Autonomous Research","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-07-31T09:14:06.207902Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.24647"},"observation_digest":"sha256:18cf6807109354c575e8e892fd664f87da03e2d9d305b1b1055fe4915d44ab9d","observation_id":"4082e15f-b500-40b7-96f1-5feb288df4fa","resolution":{"observed_at":"2026-07-31T09:14:06.207902Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-01T01:16:43.437896Z","title":"arXiv preprint arXiv:2411.15114 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.25891","last_updated":"2026-07-28T15:50:19Z","snapshot_observed_at":"2026-08-15T05:25:30.837764Z","submitted_at":"2026-07-28T15:50:19Z","title":"Messier: A High-Resolution Corpus for Cross-Benchmark Agent Evaluation","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-01T01:16:43.437896Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.25891"},"observation_digest":"sha256:3c759e2f5c25217297525ff98fe50b2369ccbecb48b2a370b7bc9afdef6829b1","observation_id":"d6f79981-5a6c-4191-b434-dd3ce80357e7","resolution":{"observed_at":"2026-08-01T01:16:43.437896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-07-31T22:25:22.349139Z","title":"arXiv preprint arXiv:2411.15114 , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.27951","last_updated":"2026-07-30T09:59:08Z","snapshot_observed_at":"2026-08-15T17:44:46.059265Z","submitted_at":"2026-07-30T09:59:08Z","title":"Safeguards Based on Copyable Context Cannot Provide Reliable Safety for LLMs","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-07-31T22:25:22.349139Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.27951"},"observation_digest":"sha256:470a87e5b3a508966db6266ddbf2946b3239a0183fee14e73c082fa879bec671","observation_id":"d0bf2f4c-2256-4be0-95f0-6a8f25daeea7","resolution":{"observed_at":"2026-07-31T22:25:22.349139Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-03T01:30:17.866119Z","title":"arXiv preprint arXiv:2411.15114 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.28887","last_updated":"2026-07-30T23:07:51Z","snapshot_observed_at":"2026-08-05T23:12:05.011998Z","submitted_at":"2026-07-30T23:07:51Z","title":"To Add Is Machine, To Delete Is Human: Measuring and Mitigating Deletion Avoidance in LLM Code Editing","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-03T01:30:17.866119Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2607.28887"},"observation_digest":"sha256:3a190f5903f150ff33446eed7d6f5cf1411363c26297ded3bb4e208d2079a693","observation_id":"c77e66a2-c16c-433a-b39f-7b6101b3ccfc","resolution":{"observed_at":"2026-08-03T01:30:17.866119Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-06T04:16:45.749872Z","title":"arXiv preprint arXiv:2411.15114 , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.05144","last_updated":"2026-08-05T17:58:58Z","snapshot_observed_at":"2026-08-10T10:36:02.288031Z","submitted_at":"2026-08-05T17:58:58Z","title":"Argus: A General-Purpose Agentic Runtime for Long-Horizon Reasoning","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-06T04:16:45.749872Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.05144"},"observation_digest":"sha256:5d3302530c8dbacf002bbc6814e4d17aa64d955e16150bc4e0d658f9080ae424","observation_id":"a0cc8a4e-0e7d-46b8-a1fd-d9b77f400289","resolution":{"observed_at":"2026-08-06T04:16:45.749872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-07T23:25:20.492888Z","title":"Re-bench: Evaluating frontier ai r&d capabilities of language model agents against human experts.arXiv preprint arXiv:2411.15114,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.05797","last_updated":"2026-08-06T09:33:09Z","snapshot_observed_at":"2026-08-15T01:27:28.704078Z","submitted_at":"2026-08-06T09:33:09Z","title":"Predicting Task Difficulty Without Rollouts","version":1},"reference_index":1978,"source":"pdf_text","source_observed_at":"2026-08-07T23:25:20.492888Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.05797"},"observation_digest":"sha256:699d4bd86bce87390c871b7e5b662208fd1b22258c7938054cd19acb69004c63","observation_id":"fd24008f-7d94-46d1-b45c-723447228ba8","resolution":{"observed_at":"2026-08-07T23:25:20.492888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-07T06:02:25.759327Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.06301","last_updated":"2026-08-06T17:21:05Z","snapshot_observed_at":"2026-08-12T16:49:55.368455Z","submitted_at":"2026-08-06T17:21:05Z","title":"HarnessOpt-Bench: Evaluating LLMs at Harness Optimization","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T06:02:25.759327Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.06301"},"observation_digest":"sha256:1f0ccabce74ff1242802154dd571df80c35a997dccd14e9d1d37328075d7a37d","observation_id":"c3ee986a-611a-463a-941e-2e856cb77cad","resolution":{"observed_at":"2026-08-07T06:02:25.759327Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-11T00:24:44.676649Z","title":"arXiv preprint arXiv:2411.15114 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.07743","last_updated":"2026-08-07T20:21:13Z","snapshot_observed_at":"2026-08-15T10:32:35.120769Z","submitted_at":"2026-08-07T20:21:13Z","title":"QuantumMind: Constraint-Grounded Agentic Reasoning for Speedup Analysis in Quantum Computing","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-11T00:24:44.676649Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.07743"},"observation_digest":"sha256:6e9fdfcff532a5bb8fa2d1aa26965c2577eb92554837f265e8a200e5066c3f6b","observation_id":"b1c4f2d5-3f9b-4e58-a2ea-052cc44da010","resolution":{"observed_at":"2026-08-11T00:24:44.676649Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-11T23:44:24.561432Z","title":"Hjalmar Wijk, Tao Lin, Joel Becker, Sami Jawhar, Neev Parikh, Thomas Broadley, Lawrence Chan, Michael Chen, Josh Clymer, Jai Dhyani, et al","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.09096","last_updated":"2026-08-11T02:53:55Z","snapshot_observed_at":"2026-08-15T06:41:00.309669Z","submitted_at":"2026-08-10T03:49:28Z","title":"Evo-Bench: Can Language Models Improve Agent Harness?","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-11T23:44:24.561432Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.09096"},"observation_digest":"sha256:5dbe6efe76a104ae59c9fa7b626f5904d5908ecd87f82ea23db77f56825c3642","observation_id":"06f8b810-3b52-4410-a6bc-6f129843f1ff","resolution":{"observed_at":"2026-08-11T23:44:24.561432Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-14T04:22:26.843491Z","title":"Hjalmar Wijk, Tao Lin, Joel Becker, Sami Jawhar, Neev Parikh, Thomas Broadley, Lawrence Chan, Michael Chen, Josh Clymer, Jai Dhyani, et al","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.09096","last_updated":"2026-08-11T02:53:55Z","snapshot_observed_at":"2026-08-15T06:41:00.309669Z","submitted_at":"2026-08-10T03:49:28Z","title":"Evo-Bench: Can Language Models Improve Agent Harness?","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-14T04:22:26.843491Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.09096"},"observation_digest":"sha256:7371aeff7a20983d8283bacca281b2591ec7f359aa32fb7a7b886ae9de8a7ba1","observation_id":"40910431-7f02-4f95-ab99-34b730e701de","resolution":{"observed_at":"2026-08-14T04:22:26.843491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15114","snapshot_observed_at":"2026-08-15T17:44:14.918108Z","title":"2411.15114 , archiveprefix =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.13060","last_updated":"2026-08-13T10:23:11Z","snapshot_observed_at":"2026-08-16T05:11:13.383877Z","submitted_at":"2026-08-13T10:23:11Z","title":"VALG: An Agentic System for ML Theory Research","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-15T17:44:14.918108Z"},"links":{"cited_paper":"/paper/2411.15114","citing_paper":"/paper/2608.13060"},"observation_digest":"sha256:dc2881a46294036cea3f1b964491204f2f93653e2f6a970f7b99454de166d56e","observation_id":"b631cdce-8a01-4068-a59f-eb8bc494689e","resolution":{"observed_at":"2026-08-15T17:44:14.918108Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2411.15114/citation-record","integrity":"/paper/2411.15114/integrity","json":"/paper/2411.15114/citation-record.json","paper":"/paper/2411.15114"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-12T14:33:13.069528Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.069528Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:b937ae987d237c13d46a9fba119845e82d6f2c964e06dec5f28a381f38c84c73","observation_id":"93e50550-a810-40fc-a4a7-8283917db63c","resolution":{"observed_at":"2026-08-12T14:33:13.069528Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.073102Z","title":"Competition-level code generation with AlphaCode","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.073102Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:8ecbf3d8515d25121a6c8642fbc19d52b05e2bc107a9b2f9ae099129fcf2bb3e","observation_id":"9214f6e2-9035-4c84-9c81-1fd6d4366a98","resolution":{"observed_at":"2026-08-12T14:33:13.073102Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.11644","last_updated":"2023-10-02T06:12:30Z","snapshot_observed_at":"2026-08-13T11:19:55.436754Z","submitted_at":"2023-06-20T16:14:25Z","title":"Textbooks Are All You Need","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.11644","snapshot_observed_at":"2026-08-12T14:33:13.076189Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.076189Z"},"links":{"cited_paper":"/paper/2306.11644","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:49a4c55c8860c1ff4301a301a51b8c89b6b22be103b91c987f90a3373a2b9a3d","observation_id":"d68dc253-f549-4593-aebf-c77561f0f0ee","resolution":{"observed_at":"2026-08-12T14:33:13.076189Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-12T14:33:13.078973Z","title":"The Llama 3 herd of models (2024)","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.078973Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:1162a5ad9d29fa5eeb9110ea128438ebfaac15bae098f585f82c6d8bb3188e5a","observation_id":"bea0344b-a812-44b7-aef2-106eb00a4911","resolution":{"observed_at":"2026-08-12T14:33:13.078973Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.701371Z","title":null,"venue":null,"work_id":"69dece1c-ca07-4beb-832a-a41d19a5bc04","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.081910Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:7c452881ff769c54529ce262ea13d0a2db828a7687164edf5d370dfb85d852c4","observation_id":"419e9304-59a9-477e-88c0-8b9dfed02aa1","resolution":{"observed_at":"2026-08-12T14:33:13.703896Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.16741","last_updated":"2025-04-18T18:14:31Z","snapshot_observed_at":"2026-08-14T17:59:52.524740Z","submitted_at":"2024-07-23T17:50:43Z","title":"OpenHands: An Open Platform for AI Software Developers as Generalist Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.16741","snapshot_observed_at":"2026-08-12T14:33:13.084827Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.084827Z"},"links":{"cited_paper":"/paper/2407.16741","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:6b2887b127145c07c5d5fff950b4408949de3e3a6d6ca6975477d7639e3d9add","observation_id":"2f4af9d2-ff40-4e0e-9bec-fe388e460293","resolution":{"observed_at":"2026-08-12T14:33:13.084827Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.695459Z","title":"Interviewing AI researchers on automation of AI r&d (2024)","venue":null,"work_id":"68aa3d0a-4ebf-4395-bc07-ebf6b6ba5705","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.088517Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:18c1d2106a4eb059ea9a6e8da883d28f819b068dcb8186ccffd1b3471d11c9a8","observation_id":"fb9d432a-3125-41da-b3f2-89fed418c256","resolution":{"observed_at":"2026-08-12T14:33:13.697544Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.2139/ssrn.4814445","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.249734Z","title":null,"venue":null,"work_id":"91ebda86-a32d-43ae-ae95-3ad42c4c77e0","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.090814Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:a0ff649848e5eba3ff5401bca1bb2f4a189d034593f8dc80ed4a2a209164ee0e","observation_id":"d94f04f1-6f4d-4ba2-b655-c0a96dd5ba44","resolution":{"observed_at":"2026-08-12T14:33:13.252997Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11690","last_updated":"2024-07-15T06:01:22Z","snapshot_observed_at":"2026-08-13T10:08:50.715798Z","submitted_at":"2023-09-20T23:45:14Z","title":"Explosive growth from AI automation: A review of the arguments","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11690","snapshot_observed_at":"2026-08-12T14:33:13.093306Z","title":"& Besiroglu, T","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.093306Z"},"links":{"cited_paper":"/paper/2309.11690","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:3e4c951a38250af4337a286f78b54693256f952e2b565802cd2f6fd968514ff7","observation_id":"13b94930-6795-4441-aff6-0483157c500f","resolution":{"observed_at":"2026-08-12T14:33:13.093306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.689540Z","title":"OpenAI preparedness framework (beta)","venue":null,"work_id":"c415c980-94bf-456f-ac79-a290f47fb456","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.095724Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:6f82b23d72ca67927841386121c65d12e44c07d8efd170ee699628a9935c9f9f","observation_id":"9108dd92-eba2-470c-b99b-80ec96d37abd","resolution":{"observed_at":"2026-08-12T14:33:13.691722Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.683074Z","title":"Frontier safety framework, version 1.0","venue":null,"work_id":"7c8a891e-a6a5-4b19-8c9f-62a7822226b3","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.097831Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:34b7210d7cfbf0ad6f33cb77233edb1815b1e56c94bc61b9d3cf4c5ab1d92e02","observation_id":"daad44ba-e91c-40f9-8909-3730c0508d4a","resolution":{"observed_at":"2026-08-12T14:33:13.685477Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.676248Z","title":"Anthropic responsible scaling policy","venue":null,"work_id":"f4b1de2c-9f61-412b-8755-997eabf286d5","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.099932Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:ff6ac55391fe124249404bc39a5142c11c645bcab465408f75d9bf7b0c46f058","observation_id":"caa42555-43ef-46bc-9dda-6754402a07df","resolution":{"observed_at":"2026-08-12T14:33:13.678875Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.669239Z","title":"Recital 110 of the eu artificial intelligence act (2024)","venue":null,"work_id":"77b946bb-063a-479a-9e67-de1a09db863a","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.101999Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:de5b559afd8229370a12b7d7e2495148f152fc626d3484e9e94d1c7c5e480d37","observation_id":"cfac877e-9688-40de-b039-a5b053e7a108","resolution":{"observed_at":"2026-08-12T14:33:13.671828Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.661235Z","title":null,"venue":null,"work_id":"795eca11-33b8-422b-a5a8-a08693c57d4b","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.103890Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:ae2e02151083f0565bd24224205ac889026fc8b668065b07ca409896b40e9273","observation_id":"8825082f-4866-415c-8950-0e5c94ab0ab6","resolution":{"observed_at":"2026-08-12T14:33:13.663824Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.654314Z","title":"The bletchley declaration on AI safety (2023)","venue":null,"work_id":"6e38b1f0-f12a-4d45-97b1-7b3cf032e896","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.105870Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:c6565b175758af726e8ae49aa93f91ee99dbcbb0c5e653724de5586eb45b842a","observation_id":"a2741764-7fae-47ce-9f96-def74a3d4ac0","resolution":{"observed_at":"2026-08-12T14:33:13.656928Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.647421Z","title":"Frontier AI safety commitments, AI seoul summit 2024","venue":null,"work_id":"d865f2a1-14db-4b93-9de0-6706cc4915f1","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.107747Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:f7b14b9f459f9e7a10ada9388bbc90294e60432f66ab5147df28cdfcc3dafd76","observation_id":"2aebe51c-8d48-42b4-a261-88fb47af02e5","resolution":{"observed_at":"2026-08-12T14:33:13.650122Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.640346Z","title":"Building an early warning system for llm-aided biological threat creation (2024)","venue":null,"work_id":"9aa88bdc-afd0-4098-a862-ecac5cdb22ab","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.109734Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:2b296e1bb7439d2efb71eb06404fa51544ccfbcd82a7e266358d6e94e382100d","observation_id":"2d114eb8-0e65-4d24-81f6-aee212b8eaf4","resolution":{"observed_at":"2026-08-12T14:33:13.643125Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.01605","last_updated":"2024-09-06T18:17:07Z","snapshot_observed_at":"2026-08-13T19:07:08.021808Z","submitted_at":"2024-08-02T23:47:27Z","title":"CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.01605","snapshot_observed_at":"2026-08-12T14:33:13.112305Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.112305Z"},"links":{"cited_paper":"/paper/2408.01605","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:317872a5d433faa9bcd8e7ae61772375007112e0ce8e61b7441b3685cb506deb","observation_id":"bfcb303c-7e77-41d0-9a92-19acb60f329a","resolution":{"observed_at":"2026-08-12T14:33:13.112305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.633138Z","title":"What a compute-centric framework says about takeoff speeds (2023)","venue":null,"work_id":"005e7fad-3a4f-466f-ba2b-298df0555c6b","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.115443Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:10c3b21f5ed6c41a6a91aa688c431d2d52efac791675316f7ffadf72fbda8e37","observation_id":"21553f6f-d87d-4220-8f52-d8c10669d594","resolution":{"observed_at":"2026-08-12T14:33:13.636145Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21514","last_updated":"2024-10-28T20:34:51Z","snapshot_observed_at":"2026-08-14T17:40:34.084043Z","submitted_at":"2024-10-28T20:34:51Z","title":"Sabotage Evaluations for Frontier Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21514","snapshot_observed_at":"2026-08-12T14:33:13.117927Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.117927Z"},"links":{"cited_paper":"/paper/2410.21514","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:74b0205afc7fa7f9ead4af46ed44128133545d1960b23d5136cb3a765b7e1e7b","observation_id":"049cc47b-1736-4ae4-ba5f-dd9eddbf9ed7","resolution":{"observed_at":"2026-08-12T14:33:13.117927Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.626590Z","title":null,"venue":null,"work_id":"09993cce-58ad-4275-9e08-beda7a94544a","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.120597Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:8dc2aa381b9a8ab43585ba987c914a9bb90304085385563b1ef06eda4b7c61d3","observation_id":"43984ef0-bde5-411f-8d9a-bb6bf903d172","resolution":{"observed_at":"2026-08-12T14:33:13.628799Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.620562Z","title":null,"venue":null,"work_id":"58ee22cb-2299-45e9-baf0-0a262b9fa552","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.122748Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:30af2210b07fe3515328690a958d83e567841df2606315aa913639f1b2d881bf","observation_id":"b52f8e02-5694-430a-881c-ebe71f379032","resolution":{"observed_at":"2026-08-12T14:33:13.622601Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.614307Z","title":"Raising the bar on swe-bench verified with claude 3.5 sonnet (2024)","venue":null,"work_id":"3683b595-93a7-44c1-af4d-732492d863ba","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.125629Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:b4f8f84ff4c67f79e1337ec769236f53f4c7112333e21e104b009ed80e038ec5","observation_id":"dda5e2a4-9525-41fa-bd72-aba38ab4de70","resolution":{"observed_at":"2026-08-12T14:33:13.616633Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19522","last_updated":"2024-05-29T20:59:57Z","snapshot_observed_at":"2026-08-12T23:54:46.649319Z","submitted_at":"2024-05-29T20:59:57Z","title":"Artificial Intelligence Index Report 2024","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19522","snapshot_observed_at":"2026-08-12T14:33:13.127921Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.127921Z"},"links":{"cited_paper":"/paper/2405.19522","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:57a3feefc7a28da5904e03bab8503addf0876a2891309810b0f0b861a0d59602","observation_id":"055c5b1d-80d5-4728-9bcd-93629c551826","resolution":{"observed_at":"2026-08-12T14:33:13.127921Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.07095","last_updated":"2025-02-26T11:57:30Z","snapshot_observed_at":"2026-08-12T16:49:10.970507Z","submitted_at":"2024-10-09T17:34:27Z","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.07095","snapshot_observed_at":"2026-08-12T14:33:13.130371Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.130371Z"},"links":{"cited_paper":"/paper/2410.07095","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:d727999ecd42d66991258fe7fa5a2400d23963bbf1b864d0ad0053cea4c44018","observation_id":"26c207d8-7c32-4898-8e65-e5ff9de0db1e","resolution":{"observed_at":"2026-08-12T14:33:13.130371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.13168","last_updated":"2024-07-18T05:15:24Z","snapshot_observed_at":"2026-08-13T19:05:50.244131Z","submitted_at":"2024-07-18T05:15:24Z","title":"SciCode: A Research Coding Benchmark Curated by Scientists","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.13168","snapshot_observed_at":"2026-08-12T14:33:13.132899Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.132899Z"},"links":{"cited_paper":"/paper/2407.13168","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:e4ec4f22370e52d9bac1d2d0fe548a39201aa32cf8aaad62b881aba2e115d2ee","observation_id":"357ddd75-67dc-4f11-8700-e9dae2fa006f","resolution":{"observed_at":"2026-08-12T14:33:13.132899Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.12022","last_updated":"2023-11-20T18:57:34Z","snapshot_observed_at":"2026-08-10T12:02:35.919497Z","submitted_at":"2023-11-20T18:57:34Z","title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.12022","snapshot_observed_at":"2026-08-12T14:33:13.135584Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.135584Z"},"links":{"cited_paper":"/paper/2311.12022","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:7066c0660fd125e32371a0efded45f6971267f7e3efbec269d4e91eec2e91b86","observation_id":"1f305af8-669c-4c5f-bd9c-842163806679","resolution":{"observed_at":"2026-08-12T14:33:13.135584Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.12983","last_updated":"2023-11-21T20:34:47Z","snapshot_observed_at":"2026-08-13T10:06:26.439949Z","submitted_at":"2023-11-21T20:34:47Z","title":"GAIA: a benchmark for General AI Assistants","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.12983","snapshot_observed_at":"2026-08-12T14:33:13.138150Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.138150Z"},"links":{"cited_paper":"/paper/2311.12983","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:fa0feceaf6c11e1c76b9ee2bbbc983316c5c04dc729fd32095e76f4ac30a6994","observation_id":"1d91d83a-f5ba-4e82-9d6a-ddcc1cad57c0","resolution":{"observed_at":"2026-08-12T14:33:13.138150Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.607409Z","title":null,"venue":null,"work_id":"3a6baa14-712c-4f69-a34b-87b292ba3105","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.140723Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:ffced5d128196e2ecbdf6c6d3bffa3b4e374624fc3a7d20c3a72fe38318671a0","observation_id":"5db7ccbf-5e43-407d-99a4-bbdb04f4c4d7","resolution":{"observed_at":"2026-08-12T14:33:13.610070Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.13854","last_updated":"2024-04-16T15:13:18Z","snapshot_observed_at":"2026-08-14T11:14:55.351653Z","submitted_at":"2023-07-25T22:59:32Z","title":"WebArena: A Realistic Web Environment for Building Autonomous Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.13854","snapshot_observed_at":"2026-08-12T14:33:13.143333Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.143333Z"},"links":{"cited_paper":"/paper/2307.13854","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:66604136a37280a46aabef0b2705bbf2bbe62b38701f21352d466295054c2071","observation_id":"33f1df9b-aaf8-44c7-b2d6-80125436611e","resolution":{"observed_at":"2026-08-12T14:33:13.143333Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03302","last_updated":"2024-04-14T21:02:16Z","snapshot_observed_at":"2026-08-13T05:57:05.956099Z","submitted_at":"2023-10-05T04:06:12Z","title":"MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03302","snapshot_observed_at":"2026-08-12T14:33:13.146051Z","title":"& Leskovec, J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.146051Z"},"links":{"cited_paper":"/paper/2310.03302","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:09134d88194fa74811bf5f4ea0e962c6fd4f409084847155082ad055fe0ec9d1","observation_id":"482e5e49-c928-48db-80dd-957436abcce4","resolution":{"observed_at":"2026-08-12T14:33:13.146051Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06769","last_updated":"2024-10-07T20:19:15Z","snapshot_observed_at":"2026-08-13T06:32:08.714493Z","submitted_at":"2024-06-10T20:08:44Z","title":"DISCOVERYWORLD: A Virtual Environment for Developing and Evaluating Automated Scientific Discovery Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06769","snapshot_observed_at":"2026-08-12T14:33:13.148692Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.148692Z"},"links":{"cited_paper":"/paper/2406.06769","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:9e65303d56d8613ad079300e9409ed23a92bdbf204fe58d3bd84004484022b03","observation_id":"70053717-4393-4d76-a4f4-d829c6c4311b","resolution":{"observed_at":"2026-08-12T14:33:13.148692Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.01374","last_updated":"2024-09-02T17:11:32Z","snapshot_observed_at":"2026-08-12T22:52:54.318306Z","submitted_at":"2024-09-02T17:11:32Z","title":"H-ARC: A Robust Estimate of Human Performance on the Abstraction and Reasoning Corpus Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.01374","snapshot_observed_at":"2026-08-12T14:33:13.151881Z","title":"K., Lake, B","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.151881Z"},"links":{"cited_paper":"/paper/2409.01374","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:fa80d02c640622d9e9efc1d7ff58cfe1a0b78f0892dbae6035c47cdd7b922c38","observation_id":"ab39c1ef-c811-4eb3-9f1c-5e419341688d","resolution":{"observed_at":"2026-08-12T14:33:13.151881Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.02155","last_updated":"2022-03-04T07:04:42Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-03-04T07:04:42Z","title":"Training language models to follow instructions with human feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.02155","snapshot_observed_at":"2026-08-12T14:33:13.154509Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.154509Z"},"links":{"cited_paper":"/paper/2203.02155","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:652905cfc9fd7bb3c20328b576a5dd909f6a6b59abd31c43c0f03325e74efed1","observation_id":"7add920a-c7e6-4f45-bd36-39f67fc91252","resolution":{"observed_at":"2026-08-12T14:33:13.154509Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-16T03:49:00.703994Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-12T14:33:13.157074Z","title":"Constitutional AI: Harmlessness from AI feedback (2022)","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.157074Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:148d7a26a8a4aa61cadf81d8a9c74db46543b4ea57af1fb2c9b4a3728da5bdbd","observation_id":"9a730257-0cea-427b-94c2-65df035f2290","resolution":{"observed_at":"2026-08-12T14:33:13.157074Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11704","last_updated":"2024-08-06T22:37:06Z","snapshot_observed_at":"2026-08-12T23:40:54.718397Z","submitted_at":"2024-06-17T16:25:04Z","title":"Nemotron-4 340B Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11704","snapshot_observed_at":"2026-08-12T14:33:13.159698Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.159698Z"},"links":{"cited_paper":"/paper/2406.11704","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:72876e061120c5f97ec863460619f21cb64b609db2995853354e22a70426339d","observation_id":"36689a9d-015e-49bd-955b-1b15cf741994","resolution":{"observed_at":"2026-08-12T14:33:13.159698Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.162197Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.162197Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:381fe57b4c97f5582340eebba5c0f321859c491c20cd5db909bbe08c2666d1a4","observation_id":"be93dd10-ed34-4eaf-a0ed-340d20b151c3","resolution":{"observed_at":"2026-08-12T14:33:13.162197Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.14838","last_updated":"2023-11-16T18:02:19Z","snapshot_observed_at":"2026-08-13T12:35:05.772403Z","submitted_at":"2023-02-28T18:37:25Z","title":"EvoPrompting: Language Models for Code-Level Neural Architecture Search","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.14838","snapshot_observed_at":"2026-08-12T14:33:13.165447Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.165447Z"},"links":{"cited_paper":"/paper/2302.14838","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:95f9a8308d2585396854c738debe88ce597ecb01a1ca8c3f430d77370fdb7aaa","observation_id":"9306505f-d8a0-4dff-aa71-b43aee00c239","resolution":{"observed_at":"2026-08-12T14:33:13.165447Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.17453","last_updated":"2024-05-28T06:50:38Z","snapshot_observed_at":"2026-08-13T04:09:11.443095Z","submitted_at":"2024-02-27T12:26:07Z","title":"DS-Agent: Automated Data Science by Empowering Large Language Models with Case-Based Reasoning","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.17453","snapshot_observed_at":"2026-08-12T14:33:13.167944Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.167944Z"},"links":{"cited_paper":"/paper/2402.17453","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:9f7d268f12ecf70647ea02bd2fa98ff3cf4bb1524d089bd27244d2846e61ce29","observation_id":"4cf5c454-0308-43ae-8b62-b4ad963cb9ed","resolution":{"observed_at":"2026-08-12T14:33:13.167944Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.07440","last_updated":"2024-09-11T17:37:48Z","snapshot_observed_at":"2026-08-13T22:20:18.011644Z","submitted_at":"2024-09-11T17:37:48Z","title":"SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.07440","snapshot_observed_at":"2026-08-12T14:33:13.170581Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.170581Z"},"links":{"cited_paper":"/paper/2409.07440","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:3ff571f7c4b5a7d42b194491900e679ffdf16c509fe90d92ef6132e67f8a8b3e","observation_id":"0306d3bd-d6ea-4cfb-9ff7-ef2b4388d32f","resolution":{"observed_at":"2026-08-12T14:33:13.170581Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.06292","last_updated":"2024-09-01T00:41:18Z","snapshot_observed_at":"2026-07-06T18:59:43.564435Z","submitted_at":"2024-08-12T16:58:11Z","title":"The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.06292","snapshot_observed_at":"2026-08-12T14:33:13.173140Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.173140Z"},"links":{"cited_paper":"/paper/2408.06292","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:5b544c464064e44b69d398700189e8a402e00eae395c4a9a645a19c8617ce47a","observation_id":"97edb651-32a6-4773-b83e-73d493d1b791","resolution":{"observed_at":"2026-08-12T14:33:13.173140Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.12931","last_updated":"2024-04-30T21:35:53Z","snapshot_observed_at":"2026-08-02T10:40:03.816188Z","submitted_at":"2023-10-19T17:31:01Z","title":"Eureka: Human-Level Reward Design via Coding Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.12931","snapshot_observed_at":"2026-08-12T14:33:13.176588Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.176588Z"},"links":{"cited_paper":"/paper/2310.12931","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:b0805398acfab605a4c34f5669a823acb24d87ad5149f8147b7d480eecfc8131","observation_id":"d4886c2d-9919-4e81-a473-72b7b55735e7","resolution":{"observed_at":"2026-08-12T14:33:13.176588Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.15568","last_updated":"2025-02-14T14:24:59Z","snapshot_observed_at":"2026-08-12T23:58:47.608264Z","submitted_at":"2024-05-24T13:57:32Z","title":"OMNI-EPIC: Open-endedness via Models of human Notions of Interestingness with Environments Programmed in Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.15568","snapshot_observed_at":"2026-08-12T14:33:13.179246Z","title":"& Clune, J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.179246Z"},"links":{"cited_paper":"/paper/2405.15568","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:745f68cbfe376aeb898f0762cbb81e6c97a10d908e39efa2bbe5849bc05b2d5d","observation_id":"619dfc4e-82ba-4856-ba1e-e93636af51f6","resolution":{"observed_at":"2026-08-12T14:33:13.179246Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08414","last_updated":"2024-11-02T22:34:31Z","snapshot_observed_at":"2026-08-12T23:44:21.284754Z","submitted_at":"2024-06-12T16:58:41Z","title":"Discovering Preference Optimization Algorithms with and for Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08414","snapshot_observed_at":"2026-08-12T14:33:13.181972Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.181972Z"},"links":{"cited_paper":"/paper/2406.08414","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:a03d6ccc262f1cb39456512aae70b591dd786ebc7c2e2f941c6ab18da6e621fc","observation_id":"943925d1-4f81-4b0c-8197-e482e3e8d8bd","resolution":{"observed_at":"2026-08-12T14:33:13.181972Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.11451","last_updated":"2024-02-21T03:04:49Z","snapshot_observed_at":"2026-08-13T04:16:51.462224Z","submitted_at":"2024-02-18T04:19:44Z","title":"SciAgent: Tool-augmented Language Models for Scientific Reasoning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.11451","snapshot_observed_at":"2026-08-12T14:33:13.184324Z","title":"Sciagent: Tool-augmented language models for scientific reasoning (2024)","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.184324Z"},"links":{"cited_paper":"/paper/2402.11451","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:0dc55ebe15c5d4171faffc0442894d686e4e05707acd6bb45fc741999730ae0d","observation_id":"374b2181-0d01-4879-a464-a963c73c8b91","resolution":{"observed_at":"2026-08-12T14:33:13.184324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.05376","last_updated":"2023-10-02T17:03:01Z","snapshot_observed_at":"2026-08-12T21:57:52.033689Z","submitted_at":"2023-04-11T17:41:13Z","title":"ChemCrow: Augmenting large-language models with chemistry tools","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.05376","snapshot_observed_at":"2026-08-12T14:33:13.186866Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.186866Z"},"links":{"cited_paper":"/paper/2304.05376","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:0246904847c16df76e5d094e8fac603217750a9933ba9df86377bbc0f2fe84dd","observation_id":"ce8cc2a6-f7fa-485e-af96-2020ce6cf0ce","resolution":{"observed_at":"2026-08-12T14:33:13.186866Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.14656","last_updated":"2024-07-23T13:56:42Z","snapshot_observed_at":"2026-08-13T04:34:18.772084Z","submitted_at":"2024-01-26T05:33:34Z","title":"Scientific Large Language Models: A Survey on Biological & Chemical Domains","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.14656","snapshot_observed_at":"2026-08-12T14:33:13.189430Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.189430Z"},"links":{"cited_paper":"/paper/2401.14656","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:a34f2165179ec373169c6d69d55f27176f5b05c2323de2e451eb104b3e3ac7b3","observation_id":"b46ed719-5073-413b-b47a-0747638c7318","resolution":{"observed_at":"2026-08-12T14:33:13.189430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.08107","last_updated":"2024-02-21T11:18:20Z","snapshot_observed_at":"2026-08-13T11:17:53.318091Z","submitted_at":"2023-06-13T19:51:22Z","title":"AutoML in the Age of Large Language Models: Current Challenges, Future Opportunities and Risks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.08107","snapshot_observed_at":"2026-08-12T14:33:13.191545Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.191545Z"},"links":{"cited_paper":"/paper/2306.08107","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:57542637cd0481872dfa5457d6efeb7a460433ddb64cd6288bc92c11026cc6af","observation_id":"9a49e1d6-cb3f-4d06-a864-f12545dc0181","resolution":{"observed_at":"2026-08-12T14:33:13.191545Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2004.10746","last_updated":"2020-04-22T17:56:07Z","snapshot_observed_at":"2026-08-10T07:39:19.641872Z","submitted_at":"2020-04-22T17:56:07Z","title":"Chip Placement with Deep Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.10746","snapshot_observed_at":"2026-08-12T14:33:13.194082Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.194082Z"},"links":{"cited_paper":"/paper/2004.10746","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:3fc6e40eaff2db93ab99b0730843644866233160897e67816622403c518e80e0","observation_id":"d7a33066-7114-4595-95a8-d5e6ea53ad7a","resolution":{"observed_at":"2026-08-12T14:33:13.194082Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.600601Z","title":"& Aydos, G","venue":null,"work_id":"fa7a66bc-38dc-4ee7-93dc-5261cd307def","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.196795Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:28156141ef38edcd2a01ccce3980764a0c55a34a1680d8b5045593b959197f30","observation_id":"bd8aacd5-e317-4e49-aa2b-d1e8e4d17979","resolution":{"observed_at":"2026-08-12T14:33:13.603105Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.593479Z","title":"Vivaria: Open-source platform for agent evaluations (2024)","venue":null,"work_id":"afdd1d8f-1990-48d7-8bd0-402450e09b4d","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.199205Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:d736e1b2a47447a599a52589a6d971cf1fdea1b186086b57a00ecf5b777abedb","observation_id":"27a19e35-bd54-415a-8ae2-b056bd5b6fbc","resolution":{"observed_at":"2026-08-12T14:33:13.596263Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.585561Z","title":"Claude 3.5 Sonnet model card addendum (2024)","venue":null,"work_id":"a7c124f9-2387-4edc-b698-c44133a85065","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.201630Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:0d90e5c4329196f5a693887ab59196db79ed08e24031a7b1fd2a1c90cc7dd454","observation_id":"2a51f86b-eb88-4539-b1a0-ff870c907679","resolution":{"observed_at":"2026-08-12T14:33:13.589134Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.577795Z","title":"OpenAI o1 system card (2024)","venue":null,"work_id":"0f49c55a-dcbc-4af6-a405-1372915fffd1","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.204017Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:f89113eaeb23d9519db3930608f00395f4585d25715260602ee88d39985c7440","observation_id":"b537316d-67d4-46f4-a304-82e7eff5addb","resolution":{"observed_at":"2026-08-12T14:33:13.580503Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.570635Z","title":"AIDE: Data science automation technical report (2024)","venue":null,"work_id":"7283c43d-2470-4db5-943c-e7e7305bd6ce","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.206318Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:1ceeaa1eb3db9e89b93c302e0b5ca1f63d35a8fb959a2c191c08d233fe7d4c5c","observation_id":"b7390dc8-00a2-4757-98b3-64235c222ea9","resolution":{"observed_at":"2026-08-12T14:33:13.573410Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.562978Z","title":"Details about METR’s preliminary evaluation of OpenAI o1-preview (2024)","venue":null,"work_id":"dd395a0c-da41-4c86-8a5c-7c35e98ef4a7","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.208642Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:8ee0f32ad5a0bf3a2f3aea036a27eb26a8205cd6da5058d0f6c19aeb78eb3324","observation_id":"7693bafe-b60a-40ed-85b6-4579cff93977","resolution":{"observed_at":"2026-08-12T14:33:13.565835Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.556430Z","title":"Claude 3.5 Sonnet: Quality, performance & price analysis (2024)","venue":null,"work_id":"14880083-79a6-45ae-8ab3-4209603f9d4b","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.210938Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:dec31a58e635b22ab5ed207fdebe682eb8556d8d5eb04ef6b34b36252037d5ae","observation_id":"a4dbaa28-eb60-414f-96d8-590c9bce99bc","resolution":{"observed_at":"2026-08-12T14:33:13.558707Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.550005Z","title":null,"venue":null,"work_id":"7c4d46d8-28f4-48d5-bef4-b98de95a0e59","year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.213202Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:9e83ddae490fd8ee7130bf02ac357e336a2a5c383d5da2c645647d19a18ae034","observation_id":"4008e77c-c644-4c3c-a78f-76a5be9c7e79","resolution":{"observed_at":"2026-08-12T14:33:13.552270Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.18290","last_updated":"2024-07-29T22:26:36Z","snapshot_observed_at":"2026-08-01T16:34:38.795326Z","submitted_at":"2023-05-29T17:57:46Z","title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.18290","snapshot_observed_at":"2026-08-12T14:33:13.215554Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.215554Z"},"links":{"cited_paper":"/paper/2305.18290","citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:0e15506b858a2792f0b65ec75222c00077f049bdc41fa1f2835095a3dfa60383","observation_id":"02f7ff6e-504e-4a93-bc58-097c5a6ada3c","resolution":{"observed_at":"2026-08-12T14:33:13.215554Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.541764Z","title":"Is this score similar to what you predicted or measured yourself, or does it come as a surprise?","venue":null,"work_id":"349f3912-4875-4c50-b417-c22059bedd0d","year":2023},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.218367Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:be0b4dbc039165b9cf44ff87f8cac3061bab614d6d3a2a78c6477e1c4a87a6be","observation_id":"b023ccb1-9588-4ffa-bd1e-97c18c884f76","resolution":{"observed_at":"2026-08-12T14:33:13.545394Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.534114Z","title":null,"venue":null,"work_id":"72782439-0836-4e6c-8965-49a6f05df52e","year":null},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.222060Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:bab840f586b3772a75e552d9ce852d72632aff9c0c6f6494ae262bfc341701f6","observation_id":"46c3d2b7-a08a-4178-868e-fe1ab97cb6d9","resolution":{"observed_at":"2026-08-12T14:33:13.536710Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.526681Z","title":null,"venue":null,"work_id":"c8afe90c-860b-4136-820e-fc4bf078d1bf","year":null},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.224373Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:c8c61ce8b30a8dd6cfc5b89d77a3af70c2146d4f640cc3346bc58c3decac57fb","observation_id":"0f6e2c1e-2e63-4d88-9548-82a0c955aa1f","resolution":{"observed_at":"2026-08-12T14:33:13.529316Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.519442Z","title":null,"venue":null,"work_id":"4c04bb47-0047-4b46-86af-004a8146f34b","year":null},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.226781Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:46c5fae91dcd05e7fe67f34f661b0f61165ffbc189b8fae5ddedbbd45a3dfeb1","observation_id":"6231b3b9-ee6a-4a25-baf1-34cc040b76dd","resolution":{"observed_at":"2026-08-12T14:33:13.521984Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.511672Z","title":null,"venue":null,"work_id":"042f4ef3-9300-40bd-83f2-20361d418344","year":null},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.229141Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:572631e1b7cc72925b2e09181ff6245c16b52119aa96fd095c39a7042cab45f3","observation_id":"9a6f4ce8-f8af-4924-a545-1e699ff2598a","resolution":{"observed_at":"2026-08-12T14:33:13.514722Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:33:13.504284Z","title":"cuda\" \"cuda","venue":null,"work_id":"0113636b-99b7-4d34-bf50-0241a16a4fb8","year":null},"citing_paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","version":2},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-12T14:33:13.231482Z"},"links":{"citing_paper":"/paper/2411.15114"},"observation_digest":"sha256:2a74e5f78d51f482f4203ade7d70f166e04ce16bec0e553101e4af7321ae5fb6","observation_id":"f6629272-2716-425d-b21e-14db69ae21ed","resolution":{"observed_at":"2026-08-12T14:33:13.507078Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.15114","last_updated":"2025-05-27T03:32:23Z","latest_version":2,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-12T16:49:37.896696Z","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts"},"reference_resolution":{"displayed":64,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":44,"verified_exact":1,"verified_fuzzy":17},"total_outbound_references":64},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 16 August 2026, this Paper Citation Record lists 64 of 64 outbound references and 62 inbound Pith citation observations for arXiv:2411.15114."}