{"as_of":"2026-08-12T12:18:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:dcf800d7d4cd7659c0382cbfba32a98efad438b4a004c64c952507dd1cdd1e07","coverage":[{"denominator":36,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":36,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T19:04:26.024098Z","state":"measured"},{"denominator":47,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":47,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":11,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":11,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T23:25:46.325101Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T08:17:45.313006Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2506.03535","last_updated":"2026-04-20T04:02:41Z","snapshot_observed_at":"2026-08-11T10:26:42.867757Z","submitted_at":"2025-06-04T03:31:00Z","title":"Across Programming Language Silos: A Study on Cross-Lingual Retrieval-augmented Code Generation","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-05-19T11:52:44.990234Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2506.03535"},"observation_digest":"sha256:e39e73c4a67bb0a8bb33414dce690407f17c42a2afc3a6dde7604fe403edf432","observation_id":"beede259-c224-4d6f-b773-850b7dbae6e6","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-08-06T23:25:46.325101Z","title":"R., and Cheung, S.-C","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.325101Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:2246099cdf47c204f3735658f0071d064c57f412ce81df8bea606e6075cfc282","observation_id":"3973a4fa-89d9-446d-a73e-f298d673980b","resolution":{"observed_at":"2026-08-06T23:25:46.325101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-08-06T20:24:20.949000Z","title":"How should i build a benchmark? arXiv preprint arXiv:2501.10711, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.02825","last_updated":"2025-08-07T06:58:08Z","snapshot_observed_at":"2026-08-09T13:14:30.968472Z","submitted_at":"2025-07-03T17:35:31Z","title":"Establishing Best Practices for Building Rigorous Agentic Benchmarks","version":5},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T20:24:20.949000Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2507.02825"},"observation_digest":"sha256:03a86bd1c869cfb5f876796173b2a2554d8039a74fc41a415289b66ecb768a61","observation_id":"58917dc5-a8db-4654-8d3a-2a65a00bfc8e","resolution":{"observed_at":"2026-08-06T20:24:20.949000Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e60cd5fdc959b4cdd86bf3af0a2ed3db0387919692d1ea79be71a4ce40ee7218","observation_id":"6a12dde6-efe5-4dfd-a678-8026e8bc1c33","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2508.15503","last_updated":"2026-05-21T21:46:55Z","snapshot_observed_at":"2026-07-06T22:16:05.197728Z","submitted_at":"2025-08-21T12:30:30Z","title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","version":5},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-18T22:02:36.307598Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2508.15503"},"observation_digest":"sha256:ade4f2f15b1b315fc49bad4af5ba741eac38a8bc8c8bd0af49c69f6949d7bcf3","observation_id":"fda34f98-829b-403e-be0e-948799f4c56c","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2508.15503","last_updated":"2026-05-21T21:46:55Z","snapshot_observed_at":"2026-07-06T22:16:05.197728Z","submitted_at":"2025-08-21T12:30:30Z","title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","version":6},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-25T08:18:18.448122Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2508.15503"},"observation_digest":"sha256:cffb67b89c40654e2c60736d3804c771356a16c1a8355df9b70ee0d255f9a7c8","observation_id":"65779e12-7cd7-49cc-bd71-1f333fd5523c","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2512.00380","last_updated":"2026-04-30T09:17:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-11-29T08:13:54Z","title":"Knowledge-Graph-Driven Data Synthesis for Low-Resource Software Development: A HarmonyOS Case Study","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-05-17T03:38:40.078229Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2512.00380"},"observation_digest":"sha256:a7ac75159beff4a57a6dd3930668e48835f14d7ff2b93d572e0c99f6b1f510c8","observation_id":"6631b1be-bbee-457c-9245-9f7bb63248dd","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2604.05100","last_updated":"2026-04-06T18:59:42Z","snapshot_observed_at":"2026-08-11T06:50:03.234044Z","submitted_at":"2026-04-06T18:59:42Z","title":"Edit, But Verify: An Empirical Audit of Instructed Code-Editing Benchmarks","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-05-10T19:16:33.635516Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2604.05100"},"observation_digest":"sha256:99c561453dfbb29bb13db01b57353d3da5b42249fe653ec607f5ee802cbdc4bb","observation_id":"0061e5a0-8657-4136-9935-52d0364e35f4","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2606.11166","last_updated":"2026-06-09T17:46:10Z","snapshot_observed_at":"2026-08-07T21:17:06.825949Z","submitted_at":"2026-06-09T17:46:10Z","title":"Flaws in the LLM Automation Narrative","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-06-27T10:52:36.252919Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2606.11166"},"observation_digest":"sha256:f841a3f39f8ad30a3fa03542dcb415d4072861773828a696bba77330a0572efd","observation_id":"8513986c-d294-41b6-a051-13b72123b60d","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2607.00062","last_updated":"2026-06-30T10:13:12Z","snapshot_observed_at":"2026-08-07T06:20:09.714519Z","submitted_at":"2026-06-30T10:13:12Z","title":"AlgoBench: Benchmarking Algorithmic Adaptation in Code Generation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-02T18:09:02.049387Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2607.00062"},"observation_digest":"sha256:e5ff11223682612905fd9fa25cda909014e352d9f3173b2d8b23c33e75e91a71","observation_id":"cc18cf3f-27f6-45f0-a7e8-ac73c421df8c","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-08-03T01:30:13.354497Z","title":"and Cheung, Shing-Chi , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.28887","last_updated":"2026-07-30T23:07:51Z","snapshot_observed_at":"2026-08-05T23:12:05.011998Z","submitted_at":"2026-07-30T23:07:51Z","title":"To Add Is Machine, To Delete Is Human: Measuring and Mitigating Deletion Avoidance in LLM Code Editing","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-03T01:30:13.354497Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2607.28887"},"observation_digest":"sha256:a19598c8fbe7742325c069d6236ecb7d93c2989080f500a6ecf350500f574302","observation_id":"eb8b0150-4c21-4608-81bf-219d2a213365","resolution":{"observed_at":"2026-08-03T01:30:13.354497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2501.10711/citation-record","integrity":"/paper/2501.10711/integrity","json":"/paper/2501.10711/citation-record.json","paper":"/paper/2501.10711"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-10T19:04:25.901736Z","title":"In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6-9, 2019","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.901736Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:e2c86488b7e44bc0f78341f31a2f8479c3c43adc7c92f6519a140b6239e341ba","observation_id":"34896d21-bd5d-4b53-8da1-2aee4c39f741","resolution":{"observed_at":"2026-08-10T19:04:25.901736Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.03065","last_updated":"2024-01-05T20:53:51Z","snapshot_observed_at":"2026-07-06T17:12:10.787061Z","submitted_at":"2024-01-05T20:53:51Z","title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.03065","snapshot_observed_at":"2026-08-10T19:04:25.915749Z","title":"arXiv preprint arXiv:2401.03065","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.915749Z"},"links":{"cited_paper":"/paper/2401.03065","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:e22370e243ba198a7ff24e05a702d7a939e3bd29b885749b57af9f84aacbac5a","observation_id":"307726f7-db38-4df1-aa70-44029ce64160","resolution":{"observed_at":"2026-08-10T19:04:25.915749Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11927","last_updated":"2025-02-09T08:47:20Z","snapshot_observed_at":"2026-08-09T06:22:50.277266Z","submitted_at":"2024-06-17T10:45:22Z","title":"On the Impacts of Contexts on Repository-Level Code Generation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11927","snapshot_observed_at":"2026-08-10T19:04:25.919849Z","title":"Preprint, arXiv:2406.11927","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.919849Z"},"links":{"cited_paper":"/paper/2406.11927","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:6827b9fe33b224373486ebd5ae6204d9ba5512e81f8f0873cd9417f3d57de545","observation_id":"d7fb5baa-a027-43a4-b79a-749c7f0ae816","resolution":{"observed_at":"2026-08-10T19:04:25.919849Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09374","last_updated":"2022-11-17T07:04:11Z","snapshot_observed_at":"2026-08-11T11:17:17.684093Z","submitted_at":"2022-11-17T07:04:11Z","title":"Execution-based Evaluation for Data Science Code Generation Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09374","snapshot_observed_at":"2026-08-10T19:04:25.923680Z","title":"In Forty-first International Confer- ence on Machine Learning, ICML 2024, Vienna, Aus- tria, July 21-27, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.923680Z"},"links":{"cited_paper":"/paper/2211.09374","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:104c401ee83ab8e29f2d11646cb63794b5d2e23f0c8aa1efcb642ce5c09d0f35","observation_id":"8ca4f069-33a3-4768-8b75-1b2ff9c01810","resolution":{"observed_at":"2026-08-10T19:04:25.923680Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06025","last_updated":"2024-06-10T05:15:30Z","snapshot_observed_at":"2026-08-08T05:40:22.152653Z","submitted_at":"2024-06-10T05:15:30Z","title":"RepoQA: Evaluating Long Context Code Understanding","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06025","snapshot_observed_at":"2026-08-10T19:04:25.946456Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.946456Z"},"links":{"cited_paper":"/paper/2406.06025","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:3768d6021e53f6d6b03746520d0db16c209a19377df617ebffc653825f1c435c","observation_id":"392285b3-6eaf-4498-b896-a7b3e5ae6a71","resolution":{"observed_at":"2026-08-10T19:04:25.946456Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.02806","last_updated":"2024-10-14T23:06:22Z","snapshot_observed_at":"2026-07-06T17:55:12.784422Z","submitted_at":"2024-04-03T15:20:57Z","title":"The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.02806","snapshot_observed_at":"2026-08-10T19:04:25.950216Z","title":"did you miss my comment or what?","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.950216Z"},"links":{"cited_paper":"/paper/2404.02806","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:d73e7ff219d6186acec9a7a8b343d8a6902183690907351e43b2c80893b3d0b5","observation_id":"da9dd4fe-2709-43c9-adad-ac262a9c5c87","resolution":{"observed_at":"2026-08-10T19:04:25.950216Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.01102","last_updated":"2023-04-03T16:02:00Z","snapshot_observed_at":"2026-08-10T16:49:45.092429Z","submitted_at":"2023-04-03T16:02:00Z","title":"RunBugRun -- An Executable Dataset for Automated Program Repair","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.01102","snapshot_observed_at":"2026-08-10T19:04:25.961358Z","title":"In 3rd IEEE/ACM International Work- shop on Automated Program Repair, APR@ICSE 2022, Pittsburgh, PA, USA, May 19, 2022 , pages 69–75","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.961358Z"},"links":{"cited_paper":"/paper/2304.01102","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:0e16f2755fb70f5a12053fdece3072482526198ad8dca167ad5df40e79e0f827","observation_id":"b0985be0-94b7-4b92-a3e6-5fa8d74508c8","resolution":{"observed_at":"2026-08-10T19:04:25.961358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.12990","last_updated":"2024-11-20T02:38:24Z","snapshot_observed_at":"2026-08-12T09:14:28.953175Z","submitted_at":"2024-11-20T02:38:24Z","title":"BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.12990","snapshot_observed_at":"2026-08-10T19:04:25.964826Z","title":"Preprint, arXiv:2411.12990","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.964826Z"},"links":{"cited_paper":"/paper/2411.12990","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:ced75ba50a37bfeefa3d12f37b9a6370a8ad12200b3e55ef09cfc02ea856e756","observation_id":"f4957f80-38f9-426e-86cb-37a07e3b7a1a","resolution":{"observed_at":"2026-08-10T19:04:25.964826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.330654Z","title":"ACM Comput","venue":null,"work_id":"eda47cab-9257-4156-adad-cd4510fda6b8","year":2020},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.968242Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:ad2154ad7be2960c159d88d3430dd770eba5ea91a757eb48b9654ef039a06228","observation_id":"841a835e-fbcf-427d-a7d2-e0249605e945","resolution":{"observed_at":"2026-08-10T19:04:26.334240Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.319988Z","title":"In The Tenth International Conference on Learning Representations, ICLR 2022, Virtual Event, April 25-29, 2022","venue":null,"work_id":"6d490a1e-eca3-40a9-80ab-93207bf582ea","year":2022},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.971681Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:ce062bcccaf34b42efe8ff29bc619de32337c2eaf6248fab91ffbaeec97120e6","observation_id":"e25778f0-993d-4db0-91f9-63d64667c6ec","resolution":{"observed_at":"2026-08-10T19:04:26.324022Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19073","last_updated":"2024-10-31T13:59:05Z","snapshot_observed_at":"2026-08-05T18:35:32.064339Z","submitted_at":"2024-06-27T10:43:04Z","title":"AMBROSIA: A Benchmark for Parsing Ambiguous Questions into Database Queries","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19073","snapshot_observed_at":"2026-08-10T19:04:25.974940Z","title":"In Findings of the Association for Computational Linguistics: EMNLP 2023, pages 10776–10787, Sin- gapore","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.974940Z"},"links":{"cited_paper":"/paper/2406.19073","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:46ee0a15dc9a4345dc9b8ea2e1e3b1df935297b5d61b2caedd9bc7438a9ebd06","observation_id":"5538b631-824e-4882-be67-1604f0ec4d8a","resolution":{"observed_at":"2026-08-10T19:04:25.974940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03163","last_updated":"2025-02-09T04:22:26Z","snapshot_observed_at":"2026-07-06T17:40:01.975792Z","submitted_at":"2024-03-05T17:56:27Z","title":"Design2Code: Benchmarking Multimodal Code Generation for Automated Front-End Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.03163","snapshot_observed_at":"2026-08-10T19:04:25.978610Z","title":"In The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.978610Z"},"links":{"cited_paper":"/paper/2403.03163","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:9711e9153f5d813a563eb8320df8bb4350a27fc8eb42add72b134bc1d8a743b1","observation_id":"641b435f-c8d6-4d26-aeaf-5bf8f6da3200","resolution":{"observed_at":"2026-08-10T19:04:25.978610Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1907.10701","last_updated":"2019-10-22T06:07:55Z","snapshot_observed_at":"2026-08-03T06:07:39.760319Z","submitted_at":"2019-07-24T20:18:28Z","title":"Benchmarking TPU, GPU, and CPU Platforms for Deep Learning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.10701","snapshot_observed_at":"2026-08-10T19:04:25.986001Z","title":"Preprint, arXiv:1907.10701","venue":null,"work_id":null,"year":1907},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.986001Z"},"links":{"cited_paper":"/paper/1907.10701","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:d57551e3ec540d283f35bac7387accc305ae16520e052fd4a83697e8c20de9bf","observation_id":"41fb8a0d-318b-45b7-95ab-53e4389c1f5e","resolution":{"observed_at":"2026-08-10T19:04:25.986001Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.308720Z","title":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed H","venue":null,"work_id":"f37dece7-2344-421b-8206-f45563487e87","year":2022},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.989890Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:7ee3c15d9e0857ef2a6223c0b85c5affe83b4cfc7e3fd417108640fd5e692f39","observation_id":"6c0761f5-e342-418f-a39e-575223e02759","resolution":{"observed_at":"2026-08-10T19:04:26.312967Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.07990","last_updated":"2024-05-13T17:59:22Z","snapshot_observed_at":"2026-08-05T02:36:46.291195Z","submitted_at":"2024-05-13T17:59:22Z","title":"Plot2Code: A Comprehensive Benchmark for Evaluating Multi-modal Large Language Models in Code Generation from Scientific Plots","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.07990","snapshot_observed_at":"2026-08-10T19:04:25.993387Z","title":"Chengyue Wu, Yixiao Ge, Qiushan Guo, Jiahao Wang, Zhixuan Liang, Zeyu Lu, Ying Shan, and Ping Luo","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.993387Z"},"links":{"cited_paper":"/paper/2405.07990","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:d429e0e0d840e5dc2be851619e005a3b01b7e92d04b5c97da88e501abcbdfa67","observation_id":"8871a011-9d4f-4647-b182-379e83234441","resolution":{"observed_at":"2026-08-10T19:04:25.993387Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13001","last_updated":"2025-05-17T02:52:25Z","snapshot_observed_at":"2026-07-06T19:05:05.530925Z","submitted_at":"2024-08-23T11:43:00Z","title":"CRUXEval-X: A Benchmark for Multilingual Code Reasoning, Understanding and Execution","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13001","snapshot_observed_at":"2026-08-10T19:04:25.997038Z","title":"CoRR, abs/2408.13001","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.997038Z"},"links":{"cited_paper":"/paper/2408.13001","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:c68db74010ed7969715ee0c252f69e925c455c21c6f3722a551d8f09f3723481","observation_id":"b5a330f3-866f-42e8-8471-0b3810afb14f","resolution":{"observed_at":"2026-08-10T19:04:25.997038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06918","last_updated":"2025-03-18T04:58:23Z","snapshot_observed_at":"2026-08-11T16:34:53.756901Z","submitted_at":"2024-06-11T03:19:18Z","title":"HumanEvo: An Evolution-aware Benchmark for More Realistic Evaluation of Repository-level Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06918","snapshot_observed_at":"2026-08-10T19:04:26.001569Z","title":"CoRR, abs/2406.06918","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.001569Z"},"links":{"cited_paper":"/paper/2406.06918","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:d81620c7646e9ca5cd7a759d5cbae3d314321fd2dfc4cbb1378be6d731c4433c","observation_id":"c618f911-c028-42e7-98f4-592bb192423a","resolution":{"observed_at":"2026-08-10T19:04:26.001569Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.297523Z","title":"0 1000 2000 3000 0 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 Citations Figure 10: Citation Distribution of Benchmarks Coding Task","venue":null,"work_id":"2fbe5f87-cc28-42c2-aba7-18aaf3ed698a","year":2000},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.009889Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:8608984678a51f777714a3b0365be4824393eab9f441c8cd8dbe67d566aed023","observation_id":"4bbc60dd-80a8-42d0-82c1-d7fec200b412","resolution":{"observed_at":"2026-08-10T19:04:26.301471Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.286238Z","title":"For example, Hu- manEval (Chen et al., 2021a) and MBPP (Austin et al., 2021)), class-level (i.e., a class with mul- tiple function units of code","venue":null,"work_id":"ab7bb8f6-8573-4451-b009-7fdcc49d8aa1","year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.013463Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:b2b3659cd741cc425b1a2762900f37b261cbda52cf9659f427180917f1453902","observation_id":"7144b574-bcae-4541-bb62-9410db5821aa","resolution":{"observed_at":"2026-08-10T19:04:26.290186Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.265083Z","title":"\"\" if len(dict.keys()) == 0: return False else: state =","venue":null,"work_id":"f587b7ee-f84d-4079-a9ba-e847b50a968c","year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.020641Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:18ebb24ed32f3a5a7a5beb5ede4c4abefaa556eaa1171883dc7f0c8d36598000","observation_id":"e886fbfb-412f-468a-8bd7-cdf61fa6333e","resolution":{"observed_at":"2026-08-10T19:04:26.268643Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.254331Z","title":"assert upper_ctr('PYthon') == 1","venue":null,"work_id":"0ca21946-4f97-46d3-aadc-d7c48d3a7ec5","year":2021},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.024098Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:ddcf2ed053f79811cd1103281a954d319a3904c8dcf5d711f8111b65a5544174","observation_id":"e2308edd-b6d1-4238-a348-5085acc9db28","resolution":{"observed_at":"2026-08-10T19:04:26.258180Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2203.13474","last_updated":"2023-02-27T21:26:48Z","snapshot_observed_at":"2026-07-06T12:52:16.992962Z","submitted_at":"2022-03-25T06:55:15Z","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.13474","snapshot_observed_at":"2026-08-10T19:04:25.953770Z","title":"Pengyu Nie, Rahul Banerjee, Junyi Jessy Li, Raymond J","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":294,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.953770Z"},"links":{"cited_paper":"/paper/2203.13474","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:0bbee8466cb9ec4084f4762e42547c117a283d64e9ac8cfb4181950b12f4ee0a","observation_id":"a0345340-8e1f-40de-b16c-c684c63cdb84","resolution":{"observed_at":"2026-08-10T19:04:25.953770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.16801","last_updated":"2024-06-25T18:04:23Z","snapshot_observed_at":"2026-08-10T06:21:25.126913Z","submitted_at":"2024-06-24T17:08:17Z","title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","version":2},"cited_work":{"arxiv_id":"2406.16801","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.16801","snapshot_observed_at":"2026-08-10T19:04:26.192430Z","title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","venue":"cs.CL","work_id":"7543192e-d38e-4cd5-86e9-f15444f6b8c3","year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":516,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.935489Z"},"links":{"cited_paper":"/paper/2406.16801","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:bd4c4e3ddd8a8f5c17e0adfdea8c5ad351b09ac711ec86626d32cf48da957986","observation_id":"b87d9451-8b50-4d17-aa66-36f687992189","resolution":{"observed_at":"2026-08-10T19:04:26.196889Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04925","last_updated":"2024-06-22T08:18:48Z","snapshot_observed_at":"2026-08-10T10:36:14.578965Z","submitted_at":"2024-01-10T04:37:38Z","title":"The Impact of Reasoning Step Length on Large Language Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04925","snapshot_observed_at":"2026-08-10T19:04:25.931274Z","title":"Mingsheng Jiao, Tingrui Yu, Xuan Li, Guanjie Qiu, Xiaodong Gu, and Beijun Shen","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":1442,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.931274Z"},"links":{"cited_paper":"/paper/2401.04925","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:aaeb05c8582209e9e9445dcf6bc9cf1bbda3da43fa46bab703e5dc8b83e177b0","observation_id":"6e9b65b5-40ce-4d6e-bbf6-3c801369c065","resolution":{"observed_at":"2026-08-10T19:04:25.931274Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15334","last_updated":"2023-05-24T16:48:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-05-24T16:48:11Z","title":"Gorilla: Large Language Model Connected with Massive APIs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15334","snapshot_observed_at":"2026-08-10T19:04:25.957389Z","title":"Wonseok Oh and Hakjoo Oh","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":1569,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.957389Z"},"links":{"cited_paper":"/paper/2305.15334","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:212ea18b6906a0bd5d04e7ab02c6d2c1cc6df63b710cecde6fb461ac9bc9c965","observation_id":"24b41b94-ab75-4ba6-a038-b2b2c6862b19","resolution":{"observed_at":"2026-08-10T19:04:25.957389Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04379","last_updated":"2024-06-06T00:06:50Z","snapshot_observed_at":"2026-08-12T10:31:47.201277Z","submitted_at":"2024-06-06T00:06:50Z","title":"VHDL-Eval: A Framework for Evaluating Large Language Models in VHDL Code Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04379","snapshot_observed_at":"2026-08-10T19:04:25.982237Z","title":"In 30th IEEE International Conference on Software Maintenance and Evolution, Victoria, BC, Canada, September 29 - October 3, 2014, pages 476–480","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2014,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.982237Z"},"links":{"cited_paper":"/paper/2406.04379","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:f2db916389dd7d709fcfe2f9ddb4e4a132b346774c1db95616a6f74af94b9cd5","observation_id":"9220f3cd-78d5-4726-b8e8-7a63997dcb24","resolution":{"observed_at":"2026-08-10T19:04:25.982237Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1709.00103","last_updated":"2017-11-09T23:06:14Z","snapshot_observed_at":"2026-08-03T05:45:09.122112Z","submitted_at":"2017-08-31T23:12:15Z","title":"Seq2SQL: Generating Structured Queries from Natural Language using Reinforcement Learning","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1709.00103","snapshot_observed_at":"2026-08-10T19:04:26.005585Z","title":"CoRR, abs/1709.00103","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.005585Z"},"links":{"cited_paper":"/paper/1709.00103","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:c85c3867b3a22431b39d771d70d9353526999d9ec163763abef102ea493f4cd3","observation_id":"8db7c4e9-ea1e-4336-94e0-7ecb703eb50f","resolution":{"observed_at":"2026-08-10T19:04:26.005585Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.341255Z","title":"Dianshu Liao, Shidong Pan, Xiaoyu Sun, Xiaoxue Ren, Qing Huang, Zhenchang Xing, Huan Jin, and Qinying Li","venue":null,"work_id":"2e9ed6ed-e9ba-4c84-be0c-f3751471b4de","year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.943198Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:e1c06dc191748833dc764f5b6e13aed060cea91e11a57f623e9f41544eb93508","observation_id":"7457eae2-0314-4877-87a8-8e5fa287585b","resolution":{"observed_at":"2026-08-10T19:04:26.345346Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.381371Z","title":null,"venue":null,"work_id":"46988a9e-89e0-45ab-af13-1c4846723e4d","year":2019},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.893263Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:c2cdf00a21c493311382d8f64d2bd2139e036d3c6ac7eb0dc36176310ea5d6c0","observation_id":"c77e4a68-9820-48cf-837b-9db14ee61242","resolution":{"observed_at":"2026-08-10T19:04:26.384535Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.390589Z","title":"In Proceedings of the 28th International Confer- ence on Computational Linguistics, COLING 2020, Barcelona, Spain (Online), December 8-13, 2020 , pages 26–38","venue":null,"work_id":"26bef4f7-405b-469d-ac73-21473f4b1876","year":2020},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.888664Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:ad2550ab7afb09baff52ec1d325e17cde32d47aec249c288b328cad29a87eb7d","observation_id":"bc2a4932-c16a-47b3-844d-063958e4e60f","resolution":{"observed_at":"2026-08-10T19:04:26.394682Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.275292Z","title":"measure the ability of these models to synthesize short Python programs from natural language descriptions","venue":null,"work_id":"0fc624a0-40ee-4160-9594-39c177fae453","year":2021},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:26.016874Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:82886987bbe469026b149dd138c955098f654510a1aeec0fbaa048a9432fb38e","observation_id":"5439fe26-5714-40ce-9e05-b0c5bb2310f4","resolution":{"observed_at":"2026-08-10T19:04:26.279136Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1807.06756","last_updated":"2021-01-12T00:04:44Z","snapshot_observed_at":"2026-07-06T06:50:50.677421Z","submitted_at":"2018-07-18T03:26:39Z","title":"SySeVR: A Framework for Using Deep Learning to Detect Software Vulnerabilities","version":3},"cited_work":{"arxiv_id":"1807.06756","doi":null,"metadata_source":"pith","pith_arxiv_id":"1807.06756","snapshot_observed_at":"2026-08-10T19:04:26.175948Z","title":"SySeVR: A Framework for Using Deep Learning to Detect Software Vulnerabilities","venue":"cs.LG","work_id":"1ff5fd15-afea-4795-acec-2044264c6052","year":2018},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.939487Z"},"links":{"cited_paper":"/paper/1807.06756","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:3c645559f60eb674981f0c8b2c1a69f92de0beb11c8bb209f6846e65e79c6f7a","observation_id":"9e39f514-2992-4811-9d48-abf6cf58bbcd","resolution":{"observed_at":"2026-08-10T19:04:26.181938Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.352024Z","title":"In 45th IEEE/ACM International Conference on Software Engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023, pages 1430–","venue":null,"work_id":"aa3207fa-57af-49ec-8a09-4fd32bf98564","year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.927478Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:1dd8005cf75e18cf9c2f2fa662d566ac061f0efd34dc32ff4d3d78dcdc88e5c0","observation_id":"cc6295b4-01b7-4629-b9f5-3c910c840f3c","resolution":{"observed_at":"2026-08-10T19:04:26.356005Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11612","last_updated":"2024-06-17T14:58:29Z","snapshot_observed_at":"2026-07-06T18:32:15.404516Z","submitted_at":"2024-06-17T14:58:29Z","title":"Long Code Arena: a Set of Benchmarks for Long-Context Code Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11612","snapshot_observed_at":"2026-08-10T19:04:25.910216Z","title":"O’Reilly Media, Inc","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.910216Z"},"links":{"cited_paper":"/paper/2406.11612","citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:923a72789a5dd93dbd638bdbf9c55562e27522825088f6bf9dce0a6a27488d2f","observation_id":"b154828c-19f1-4b79-84bf-0bb7265d9243","resolution":{"observed_at":"2026-08-10T19:04:25.910216Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.372414Z","title":"Lakshya A","venue":null,"work_id":"f8985600-4361-4828-9200-728e9901abbc","year":2023},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":5445,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.897709Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:491c616c4514b212c32ae46483e1bd75e1ae30dbbd60fac7d7c13d1cd34caa7b","observation_id":"43d11c34-c450-4efa-a791-c2a118d34726","resolution":{"observed_at":"2026-08-10T19:04:26.375676Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T19:04:26.362623Z","title":"Ramakrishna Bairi, Atharv Sonwane, Aditya Kanade, Vageesh D","venue":null,"work_id":"38bd873d-bc77-42ee-bdf4-71ff63e0eb9f","year":2024},"citing_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"reference_index":8474,"source":"pdf_text","source_observed_at":"2026-08-10T19:04:25.906189Z"},"links":{"citing_paper":"/paper/2501.10711"},"observation_digest":"sha256:df37c13387000c6852432e4be4a2497c85b8c61e983911d91c1de35d80a745e0","observation_id":"35c6a7bb-d39d-49b3-a7d7-83b04d35f4e5","resolution":{"observed_at":"2026-08-10T19:04:26.366160Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","latest_version":5,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility"},"reference_resolution":{"displayed":36,"state_counts":{"malformed_identifier":0,"metadata_mismatch":2,"parse_uncertain":0,"unresolved":21,"verified_exact":0,"verified_fuzzy":13},"total_outbound_references":36},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 12 August 2026, this Paper Citation Record lists 36 of 36 outbound references and 11 inbound Pith citation observations for arXiv:2501.10711."}