{"as_of":"2026-08-09T14:05:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c462b2f50bb4b05e2bb895aa8fa427147afc5e4255c76287ee160a81c43a6194","coverage":[{"denominator":67,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":67,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T21:56:20.506734Z","state":"measured"},{"denominator":72,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":72,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":5,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":5,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T12:14:20.943931Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-17T02:05:18.872540Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.09328","snapshot_observed_at":"2026-08-07T12:14:20.943931Z","title":"Copilot arena: A platform for code LLM evaluation in the wild","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.00204","last_updated":"2025-05-30T20:19:39Z","snapshot_observed_at":"2026-08-08T01:14:12.594755Z","submitted_at":"2025-05-30T20:19:39Z","title":"Structure-Aware Fill-in-the-Middle Pretraining for Code","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-07T12:14:20.943931Z"},"links":{"cited_paper":"/paper/2502.09328","citing_paper":"/paper/2506.00204"},"observation_digest":"sha256:e2672fb88a9605f42013cfabe1190c8c7035ca879ac42c61002369cbf182e29b","observation_id":"8e9c250b-57ee-442f-9b79-f5c9ba4c29c3","resolution":{"observed_at":"2026-08-07T12:14:20.943931Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"cited_work":{"arxiv_id":"2502.09328","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2502.09328","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Copilot arena: A platform for code llm evaluation in the wild.arXiv preprint arXiv:2502.09328","venue":null,"work_id":"fc231868-f448-4ef3-ac00-3b3e0e651ae5","year":2025},"citing_paper":{"arxiv_id":"2506.17298","last_updated":"2025-06-17T17:06:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-06-17T17:06:18Z","title":"Mercury: Ultra-Fast Language Models Based on Diffusion","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-17T02:05:18.834924Z"},"links":{"cited_paper":"/paper/2502.09328","citing_paper":"/paper/2506.17298"},"observation_digest":"sha256:7a764d575862f337add98bf2dc6f8821d7713f70d39ddc6fd0bb9ef52e2e414b","observation_id":"fc242de9-59c8-4e7b-b077-f0259d48190f","resolution":{"observed_at":"2026-05-17T02:05:18.874684Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.09328","snapshot_observed_at":"2026-08-03T06:55:31.047609Z","title":"j wins” vs. “k wins","venue":null,"work_id":null,"year":1998},"citing_paper":{"arxiv_id":"2601.21816","last_updated":"2026-06-06T18:57:36Z","snapshot_observed_at":"2026-08-03T14:19:02.729284Z","submitted_at":"2026-01-29T15:00:07Z","title":"Nonparametric LLM Evaluation from Preference Data","version":2},"reference_index":2007,"source":"pdf_text","source_observed_at":"2026-08-03T06:55:31.047609Z"},"links":{"cited_paper":"/paper/2502.09328","citing_paper":"/paper/2601.21816"},"observation_digest":"sha256:4a3b20a32a4f0feb20835a2098098228378bee7b7b7c06f7e6d9c6020e8ad758","observation_id":"94623b0d-e1d2-4c2f-a239-9613c934811f","resolution":{"observed_at":"2026-08-03T06:55:31.047609Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"cited_work":{"arxiv_id":"2502.09328","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2502.09328","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Copilot arena: A platform for code llm evaluation in the wild.arXiv preprint arXiv:2502.09328","venue":null,"work_id":"fc231868-f448-4ef3-ac00-3b3e0e651ae5","year":2025},"citing_paper":{"arxiv_id":"2604.05100","last_updated":"2026-04-06T18:59:42Z","snapshot_observed_at":"2026-07-06T22:53:55.926521Z","submitted_at":"2026-04-06T18:59:42Z","title":"Edit, But Verify: An Empirical Audit of Instructed Code-Editing Benchmarks","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-05-10T19:16:33.635516Z"},"links":{"cited_paper":"/paper/2502.09328","citing_paper":"/paper/2604.05100"},"observation_digest":"sha256:7585e4a97a2c3450bdfa4bb4d1e52c34edfaa36bc0dc10ca303085be2df98481","observation_id":"08ada4f0-b026-4806-a4ad-e031b0d3f71e","resolution":{"observed_at":"2026-05-10T23:15:48.220455Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"cited_work":{"arxiv_id":"2502.09328","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2502.09328","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Copilot arena: A platform for code llm evaluation in the wild.arXiv preprint arXiv:2502.09328","venue":null,"work_id":"fc231868-f448-4ef3-ac00-3b3e0e651ae5","year":2025},"citing_paper":{"arxiv_id":"2605.01104","last_updated":"2026-05-01T21:20:38Z","snapshot_observed_at":"2026-08-01T04:01:27.652769Z","submitted_at":"2026-05-01T21:20:38Z","title":"RECAP: An End-to-End Platform for Capturing, Replaying, and Analyzing AI-Assisted Programming Interactions","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-05-09T18:39:05.786210Z"},"links":{"cited_paper":"/paper/2502.09328","citing_paper":"/paper/2605.01104"},"observation_digest":"sha256:3cb5b4c4978e9bcf9beb1361ad15bfe00e24536708eac0b3115fe11fd59ef8e7","observation_id":"81ec95f4-fff1-4c73-89c7-d11343f5d94f","resolution":{"observed_at":"2026-05-11T16:06:48.354349Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2502.09328/citation-record","integrity":"/paper/2502.09328/integrity","json":"/paper/2502.09328/citation-record.json","paper":"/paper/2502.09328"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2302.06590","last_updated":"2023-02-13T18:42:46Z","snapshot_observed_at":"2026-08-09T03:51:13.580064Z","submitted_at":"2023-02-13T18:42:46Z","title":"The Impact of AI on Developer Productivity: Evidence from GitHub Copilot","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.06590","snapshot_observed_at":"2026-08-07T21:56:20.168071Z","title":"The impact of ai on developer productivity: Evidence from github copilot","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.168071Z"},"links":{"cited_paper":"/paper/2302.06590","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:a87e888cbafd25d0b327d6ccf513ac0445be85f1b61ee8795405b8c7b5086db3","observation_id":"c9694f2f-22ba-475f-8aca-298ac64193f1","resolution":{"observed_at":"2026-08-07T21:56:20.168071Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.640887Z","title":null,"venue":null,"work_id":"1fa247af-e53b-48df-92e1-296fa9bb9fd1","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.173571Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:484092e48146d85a1f7d05fb89b0d7d4991557739346b4e138dac34855e4a01c","observation_id":"1808e117-8aa5-4a54-b304-ef711dd3c846","resolution":{"observed_at":"2026-08-07T21:56:21.646007Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.626159Z","title":"Beyond the bar: Generative ai as a transformative component in legal document review","venue":null,"work_id":"af36e425-eb8c-4b59-8818-4a08551ae4f4","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.179095Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:d4c7deedba79e2a206d2e44d2eded4d84f7299ed8dbad14ca6a0b61488cc77aa","observation_id":"b24950e9-baac-4b14-be37-e8573b892b65","resolution":{"observed_at":"2026-08-07T21:56:21.631032Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.610408Z","title":"Evaluation gaps in machine learning practice","venue":null,"work_id":"06d45351-36c7-43ee-88ec-dba63065c488","year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.184080Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:75e88f8875ad3c15857bd4490116f9678758aed1d1063d61f867c34a96285b4a","observation_id":"5cde8a51-e30d-43c0-8e5f-fcec4a4c94a6","resolution":{"observed_at":"2026-08-07T21:56:21.615702Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.591981Z","title":"Benchmarks as microscopes: A call for model metrology","venue":null,"work_id":"84d5d245-1355-4dcb-a2f0-85db87f16460","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.189130Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:ecaf9216abf5366f85e7b93599e3dad47e0d39a0fad6d055bd36ce3ffc451a2c","observation_id":"2c00d2a4-52e3-48fc-8eb9-ecf6554c787c","resolution":{"observed_at":"2026-08-07T21:56:21.598549Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01502","last_updated":"2024-07-01T17:48:14Z","snapshot_observed_at":"2026-08-06T22:15:08.500389Z","submitted_at":"2024-07-01T17:48:14Z","title":"AI Agents That Matter","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01502","snapshot_observed_at":"2026-08-07T21:56:20.194237Z","title":"Ai agents that matter","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.194237Z"},"links":{"cited_paper":"/paper/2407.01502","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:77406ba5591e7e081e38d5397c6ab12ec20cc053adcf407f380f57998511674c","observation_id":"240c7314-fc22-43d1-8465-4e6b8da2d046","resolution":{"observed_at":"2026-08-07T21:56:20.194237Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.575206Z","title":"Gonzalez, and Ion Stoica","venue":null,"work_id":"650440cb-9d0f-4c4c-bf1e-d9154afa9cc7","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.199962Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:d5611c2bd8400aea46d20435b6bdf5a64bb915d2845968075239a2dc40d2b361","observation_id":"9f70086e-37b1-4b82-a41c-c9680b1320e0","resolution":{"observed_at":"2026-08-07T21:56:21.580094Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.543305Z","title":"Alpacafarm: A simulation framework for methods that learn from human feedback","venue":null,"work_id":"768fd2ec-928f-46dc-a435-7cf8b3013832","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.209620Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:563ba360e93dfad74866f137074a878a279d95b1264fe3191f4b8b18bbd843bb","observation_id":"f646da96-c0f9-4f8c-ab09-ab4e3fbd3258","resolution":{"observed_at":"2026-08-07T21:56:21.548460Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-07T21:56:20.214333Z","title":"Chatbot arena: An open platform for evaluating llms by human preference","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.214333Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:646d1d18e6fd8c3d5f45928a88aa9294abb9c33e6b8d574b15b2e3743c77d173","observation_id":"6082a519-ccdc-4720-b3b5-68e5a11a0060","resolution":{"observed_at":"2026-08-07T21:56:20.214333Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.527593Z","title":null,"venue":null,"work_id":"10265364-b9ff-4c3f-b2d5-d9507070c1e1","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.220493Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:47b338d50a00ab3ef72375a4823809c68a45189a82fbbadb259f4fae6b1b91a1","observation_id":"346acf53-775c-4ac9-a2b7-5cb7936e4c2e","resolution":{"observed_at":"2026-08-07T21:56:21.532565Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:20.225175Z","title":"Github copilot - your ai pair programmer, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.225175Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:f1d03da3c82b3072bff29dca98b9fb1749bc2015cb0fc0068bb99e981efcf8e3","observation_id":"16463f74-676b-4bd1-821f-23b542182623","resolution":{"observed_at":"2026-08-07T21:56:20.225175Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-07T21:56:20.229816Z","title":"Evalu- ating large language models trained on code","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.229816Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:88483b4d51f95506da469cba29b4ab1b3e94c9f9e5e97b36db9a8687bd2661e4","observation_id":"e5a815bb-1701-4abf-bd59-f9abdcca4e98","resolution":{"observed_at":"2026-08-07T21:56:20.229816Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-07T21:56:20.235285Z","title":"Program synthesis with large language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.235285Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:e11c71ddccd922e20e0529e92b693301b0eff5d64d8f4adaaf07f9825fd14a76","observation_id":"e8aa71b9-8cd6-4112-b3ac-31e0048363dc","resolution":{"observed_at":"2026-08-07T21:56:20.235285Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-07T21:56:20.240188Z","title":"Livecodebench: Holistic and contam- ination free evaluation of large language models for code","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.240188Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:04dbc70bea18e095cc2fed9c63e67758d4a29bfd70cf0daa7c6462f2a7d752a4","observation_id":"55e02bf3-6dba-4b3e-9b25-6057b5377ef2","resolution":{"observed_at":"2026-08-07T21:56:20.240188Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19314","last_updated":"2025-04-18T19:36:00Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-06-27T16:47:42Z","title":"LiveBench: A Challenging, Contamination-Limited LLM Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19314","snapshot_observed_at":"2026-08-07T21:56:20.245128Z","title":"Livebench: A challenging, contamination-free llm benchmark","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.245128Z"},"links":{"cited_paper":"/paper/2406.19314","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:656fca274e1f691507b277eaabb034f2c801baac58a84a3c399ecbc1b97588c3","observation_id":"f2d5671b-b15b-4c51-a418-624fb29b0a7b","resolution":{"observed_at":"2026-08-07T21:56:20.245128Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.501921Z","title":"Expectation vs","venue":null,"work_id":"7d9dfa2f-ff83-4f32-99b8-cf24d272b513","year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.250065Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:6c06a6f9aa40ae4ff81af767eebd764770ba775f8da33cdaa35f9559237b923f","observation_id":"ea8d7c12-bed4-4a89-a160-bb035c3a3a3e","resolution":{"observed_at":"2026-08-07T21:56:21.506828Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.485448Z","title":"The programmer’s assistant: Conversational interaction with a large language model for software development","venue":null,"work_id":"ddedad79-3df0-4e1f-82ec-527c099a8e26","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.254679Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:516d63faa15b245b57579d338917e357c21ec6ddc9c43841ba04263001c7a3fb","observation_id":"8a4acac1-cdb5-467f-9cee-0ecc5e202288","resolution":{"observed_at":"2026-08-07T21:56:21.491197Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.02806","last_updated":"2024-10-14T23:06:22Z","snapshot_observed_at":"2026-07-06T17:55:12.784422Z","submitted_at":"2024-04-03T15:20:57Z","title":"The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.02806","snapshot_observed_at":"2026-08-07T21:56:20.259808Z","title":"The realhumaneval: Evaluating large language models’ abilities to support programmers","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.259808Z"},"links":{"cited_paper":"/paper/2404.02806","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:d225ecb32d91df4d79053c9b66c711295b69b66a8a4531ea3bfaa08e809e28a6","observation_id":"905af829-b390-43e0-96fa-7d373ca178a1","resolution":{"observed_at":"2026-08-07T21:56:20.259808Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11069","last_updated":"2024-06-16T20:53:25Z","snapshot_observed_at":"2026-08-06T16:39:25.477229Z","submitted_at":"2024-06-16T20:53:25Z","title":"WildVision: Evaluating Vision-Language Models in the Wild with Human Preferences","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11069","snapshot_observed_at":"2026-08-07T21:56:20.264700Z","title":"Wildvision: Evaluating vision-language models in the wild with human preferences","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.264700Z"},"links":{"cited_paper":"/paper/2406.11069","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:87991a08cbdfb6f501a713725f4be919818535e8fa6e97cfc9fb8996b228db9d","observation_id":"9c531192-deec-4d34-a691-ed4fecd7659b","resolution":{"observed_at":"2026-08-07T21:56:20.264700Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.470871Z","title":"Dai, and Quoc V Le","venue":null,"work_id":"325862ca-beda-46b0-95c9-0643dbee484d","year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.269619Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:b5fb600b137b3ffe6c004e5ecc0f4d571fe7d3d038219daee11ca6d8f0c3f129","observation_id":"8c760a64-64f7-46de-b07d-c0724c261bf5","resolution":{"observed_at":"2026-08-07T21:56:21.475509Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05999","last_updated":"2023-04-09T14:31:40Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T16:25:26Z","title":"InCoder: A Generative Model for Code Infilling and Synthesis","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05999","snapshot_observed_at":"2026-08-07T21:56:20.274059Z","title":"Incoder: A generative model for code infilling and synthesis, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.274059Z"},"links":{"cited_paper":"/paper/2204.05999","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:633be7470b14c0e9355b830827fc80251286d0f6a5f3b60fbcc439fd825b8ae4","observation_id":"822731ef-f5bb-4846-89b1-67623ba84ad2","resolution":{"observed_at":"2026-08-07T21:56:20.274059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04814","last_updated":"2024-06-23T01:17:48Z","snapshot_observed_at":"2026-07-06T17:41:14.805521Z","submitted_at":"2024-03-07T05:05:56Z","title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04814","snapshot_observed_at":"2026-08-07T21:56:20.278997Z","title":"Evaluation of llms on syntax-aware code fill-in-the-middle tasks","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.278997Z"},"links":{"cited_paper":"/paper/2403.04814","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:77250ffcad52861154e3cd5467c5d3e874cc0c9c182a53d49b58a967c6300713","observation_id":"3edb20c8-079b-40e2-86b8-118afdcc51cb","resolution":{"observed_at":"2026-08-07T21:56:20.278997Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2207.14255","last_updated":"2022-07-28T17:40:47Z","snapshot_observed_at":"2026-08-07T11:38:07.397956Z","submitted_at":"2022-07-28T17:40:47Z","title":"Efficient Training of Language Models to Fill in the Middle","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.14255","snapshot_observed_at":"2026-08-07T21:56:20.284825Z","title":"Efficient training of language models to fill in the middle","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.284825Z"},"links":{"cited_paper":"/paper/2207.14255","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:5507fe8699659cd8a21e14ecc13f6045b89e88567a18a6aefad3c16e3e9bac96","observation_id":"312abebc-6725-424b-b48d-900cfeda23ff","resolution":{"observed_at":"2026-08-07T21:56:20.284825Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04770","last_updated":"2024-10-05T22:39:51Z","snapshot_observed_at":"2026-08-06T14:20:21.460707Z","submitted_at":"2024-06-07T09:15:44Z","title":"WildBench: Benchmarking LLMs with Challenging Tasks from Real Users in the Wild","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04770","snapshot_observed_at":"2026-08-07T21:56:20.290154Z","title":"Wildbench: Benchmarking llms with challenging tasks from real users in the wild","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.290154Z"},"links":{"cited_paper":"/paper/2406.04770","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:1687053c14183f87390b931df969e4d62b8200afcbadcbda6372b6c31a35544c","observation_id":"2a3788e9-ddf5-4148-b7e3-8a95edbcc856","resolution":{"observed_at":"2026-08-07T21:56:20.290154Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.455860Z","title":"Raising the bar on swe-bench verified with claude 3.5 sonnet, 2024","venue":null,"work_id":"a27482e1-c673-4e1b-a214-54aebba7fff4","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.296266Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:0f4e096e7ad5c544aa3f1525c6a45c93a796266de0a5dca0374fbb0223b969c8","observation_id":"0f491527-cc55-447c-8fe1-07393d1398f8","resolution":{"observed_at":"2026-08-07T21:56:21.460778Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:20.301371Z","title":"Rank analysis of incomplete block designs: I","venue":null,"work_id":null,"year":1952},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.301371Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:5617fa937a05efa9871c5e5284ce242e96e9825d2068440ff144f9dc4ae58df2","observation_id":"3ff4f5ef-02ae-4937-95c4-7ec20ac61431","resolution":{"observed_at":"2026-08-07T21:56:20.301371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-07-31T19:00:59.311189Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-07T21:56:20.306648Z","title":"Bigcodebench: Benchmarking code generation with diverse function calls and complex instructions, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.306648Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:5790108d1a08b719e00560ec38970e0ebd0467924306deba17631f26f8518a22","observation_id":"80d70cea-7776-4ea4-894a-4a8507497d32","resolution":{"observed_at":"2026-08-07T21:56:20.306648Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.16694","last_updated":"2024-03-24T15:41:21Z","snapshot_observed_at":"2026-08-03T09:29:36.187988Z","submitted_at":"2024-02-26T16:09:00Z","title":"HumanEval-XL: A Multilingual Code Generation Benchmark for Cross-lingual Natural Language Generalization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.16694","snapshot_observed_at":"2026-08-07T21:56:20.311466Z","title":"Humaneval-xl: A multilingual code gen- eration benchmark for cross-lingual natural language generalization","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.311466Z"},"links":{"cited_paper":"/paper/2402.16694","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:b236e3bba54cf3ad12dc1b216fce7c8b17b2c07725a84390aa8f7868495f51b1","observation_id":"47fc453b-f8fd-4c5f-b88a-65018b72466a","resolution":{"observed_at":"2026-08-07T21:56:20.311466Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.05530","last_updated":"2024-12-16T17:39:39Z","snapshot_observed_at":"2026-07-06T17:41:42.995949Z","submitted_at":"2024-03-08T18:54:20Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.05530","snapshot_observed_at":"2026-08-07T21:56:20.317151Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.317151Z"},"links":{"cited_paper":"/paper/2403.05530","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:43f37a6c8ae10b86b2d045cea03529e1c250868a75021af214245909a31c0cae","observation_id":"75aa717a-b98f-46bb-9619-e3f330a73e18","resolution":{"observed_at":"2026-08-07T21:56:20.317151Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-07T21:56:20.321916Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.321916Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:b0261ddf6605cbb83d1af12a341d6a899626930c3c9d3dd263b895f0fd2cea69","observation_id":"7e6f9726-e1cb-415b-8763-17060bc3a921","resolution":{"observed_at":"2026-08-07T21:56:20.321916Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:20.326862Z","title":"Fine-grained human feedback gives better rewards for language model training","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.326862Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:3bf316e8058e6cf9015cac3108c1ff3366a77f2ba8b4269970e47fa285f851fd","observation_id":"1341167b-ca6a-44c5-9823-7b6ac76e7ffc","resolution":{"observed_at":"2026-08-07T21:56:20.326862Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.14146","last_updated":"2022-11-17T14:12:14Z","snapshot_observed_at":"2026-08-09T12:45:08.941800Z","submitted_at":"2022-04-29T15:06:58Z","title":"Training Language Models with Language Feedback","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.14146","snapshot_observed_at":"2026-08-07T21:56:20.331588Z","title":"Training language models with language feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.331588Z"},"links":{"cited_paper":"/paper/2204.14146","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:fe2d43065dff2a1f27ba8dd796de94f5cdf8c126d6ca47a7bd85be70c7cb33c6","observation_id":"3ae649dc-bcb4-4e30-9704-7e0ef0cfd0c6","resolution":{"observed_at":"2026-08-07T21:56:20.331588Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:20.337471Z","title":"Training language models to follow instructions with human feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.337471Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:d75b5eb717107d380b30da24b7a82fe885c7cd84fcda939b4e6f2c87afd55018","observation_id":"09a3ee1f-f729-4d38-85e7-3627f55b2e82","resolution":{"observed_at":"2026-08-07T21:56:20.337471Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.08687","last_updated":"2025-03-25T22:17:42Z","snapshot_observed_at":"2026-08-08T01:13:06.347998Z","submitted_at":"2024-12-11T18:59:46Z","title":"VisionArena: 230K Real World User-VLM Conversations with Preference Labels","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.08687","snapshot_observed_at":"2026-08-07T21:56:20.342641Z","title":"Gonzalez, and Wei-Lin Chiang","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.342641Z"},"links":{"cited_paper":"/paper/2412.08687","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:f7172178554a240f33e4a3cbf896debf9a6edba6e940427381c58c71ca10a636","observation_id":"b896b753-d32f-4ac8-a8a6-34808e5425df","resolution":{"observed_at":"2026-08-07T21:56:20.342641Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.411715Z","title":"CodeXGLUE: A machine learning benchmark dataset for code understanding and generation","venue":null,"work_id":"cf5fcf96-f57d-4875-bbe2-376c34e37965","year":2021},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.348807Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:97b2f0eb7b10c4696d53c768e61e1216548dfff82358667963bbd89d81f66700","observation_id":"21c37956-7be2-4386-bdd3-8d6eb69d2a9f","resolution":{"observed_at":"2026-08-07T21:56:21.416663Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.397218Z","title":"Codegen: An open large language model for code with multi-turn program synthesis","venue":null,"work_id":"d93c042c-cc47-47a9-9d90-98b38c927c20","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.353529Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:374447d4a739436a99d984129296a20d23e8c73774e738853da65f23af4f7a99","observation_id":"d08b4ecc-dc20-4e2d-9c92-c1ac00d611f1","resolution":{"observed_at":"2026-08-07T21:56:21.402072Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.08474","last_updated":"2022-06-16T22:49:39Z","snapshot_observed_at":"2026-07-06T13:21:49.288193Z","submitted_at":"2022-06-16T22:49:39Z","title":"XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.08474","snapshot_observed_at":"2026-08-07T21:56:20.364223Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.364223Z"},"links":{"cited_paper":"/paper/2206.08474","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:4230fcce4bde173dfc3a826d352dc5cdf05f0019f81770e17dadf37ea234914f","observation_id":"2556564c-1f34-4267-9ff8-4414bd9cb7b6","resolution":{"observed_at":"2026-08-07T21:56:20.364223Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.382089Z","title":null,"venue":null,"work_id":"8bceb846-512e-48f1-8ee1-849b2b653ad3","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.358644Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:2fd000cae868a90e996712da521bf623f278bcf2f2a8d6a8143931c476461845","observation_id":"94c4e8c3-2e3c-4bb7-9042-54e0564e8843","resolution":{"observed_at":"2026-08-07T21:56:21.387056Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.351977Z","title":"Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation","venue":null,"work_id":"40095646-8d97-4e13-83f4-1ae2a5161ae3","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.374664Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:505153e2a099144174ed6a94ac5fedc3737164b164aba25a08ede65bd9982bd8","observation_id":"a3e2a942-0646-476f-9d83-bf11ac275432","resolution":{"observed_at":"2026-08-07T21:56:21.357024Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.367065Z","title":"Recode: Robustness evaluation of code generation models","venue":null,"work_id":"db800634-118c-4266-a35a-f12c27a6585d","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.368974Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:c3daec6a31cebbb2bdf2f66eaeeeb3cd4a9dfdddf9a74637ce1f356ec5a20aba","observation_id":"d8d595ac-f25a-4502-a05b-a361a7b04221","resolution":{"observed_at":"2026-08-07T21:56:21.372216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.03004","last_updated":"2023-11-06T07:16:58Z","snapshot_observed_at":"2026-07-06T14:59:03.634181Z","submitted_at":"2023-03-06T10:08:51Z","title":"xCodeEval: A Large Scale Multilingual Multitask Benchmark for Code Understanding, Generation, Translation and Retrieval","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.03004","snapshot_observed_at":"2026-08-07T21:56:20.384490Z","title":"xcodeeval: A large scale multilingual multitask benchmark for code understanding, generation, translation and retrieval","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.384490Z"},"links":{"cited_paper":"/paper/2303.03004","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:72de6d655bdef80e8ae822b832c2520e7c6e2adbaa2f684c48ba66148303b0e4","observation_id":"7257a80f-831c-4cf0-98f3-f81fb76438fa","resolution":{"observed_at":"2026-08-07T21:56:20.384490Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.336270Z","title":"Swe-bench: Can language models resolve real-world github issues? In The Twelfth International Conference on Learning Representations , 2023","venue":null,"work_id":"3f9d9bfb-cd3c-41eb-98b4-018e41f9cd34","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.379328Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:4dab8c23eeb73d941959086d45a7df10ba41aa8312312921bb800447573428ef","observation_id":"0badcd11-4377-4db8-a58c-05d692fc7401","resolution":{"observed_at":"2026-08-07T21:56:21.341503Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.321577Z","title":"Multipl-e: a scalable and polyglot approach to benchmarking neural code generation","venue":null,"work_id":"eaa72e20-cca1-492a-981d-85b57f742e03","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.394446Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:8ba4113b288d40adb38fcf43872b0b99495bc38f71a2fc7cdaadbf146d4d28ac","observation_id":"86d980e2-f0dd-4c55-94a4-e19944a2c6d7","resolution":{"observed_at":"2026-08-07T21:56:21.326409Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.08588","last_updated":"2024-06-07T04:34:16Z","snapshot_observed_at":"2026-07-06T16:47:41.767386Z","submitted_at":"2023-11-14T23:18:52Z","title":"CodeScope: An Execution-based Multilingual Multitask Multidimensional Benchmark for Evaluating LLMs on Code Understanding and Generation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.08588","snapshot_observed_at":"2026-08-07T21:56:20.389296Z","title":"Codescope: An execution-based multilingual multitask multidimensional benchmark for evaluating llms on code understanding and generation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.389296Z"},"links":{"cited_paper":"/paper/2311.08588","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:9e416c91bf662ba405a378f699bb23325f4298abfbd3c1f6eb01a54c703c2ffe","observation_id":"c4398e9e-b76f-4cea-9e54-431709172ac7","resolution":{"observed_at":"2026-08-07T21:56:20.389296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.292272Z","title":"Large language models of code fail at completing code with potential bugs","venue":null,"work_id":"5627e7ae-2f96-4a70-8530-cf762b50cf57","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.403936Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:e4f4bcc04c640c9ca98d6ce27bdef291853e716ef8780c476c7fb714ea517729","observation_id":"c79ed065-9a12-441f-95ae-8b7b01d4bc71","resolution":{"observed_at":"2026-08-07T21:56:21.297005Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.307063Z","title":"Octopack: Instruction tuning code large language models","venue":null,"work_id":"36edb408-30cf-493f-b73b-1e9ae3dc86d5","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.399150Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:257b93b69dbea9c5db0312d853372f8e42185735325b2b1f3258501ca964ea29","observation_id":"5d8e2407-d303-453f-836f-8d103848fdce","resolution":{"observed_at":"2026-08-07T21:56:21.311951Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.262410Z","title":"R2e: Turning any github repository into a programming agent environment","venue":null,"work_id":"224e62da-2a65-456d-b362-f0f16c37ee93","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.414718Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:05d5af6ce2f135c2862edea9b520a972866fe7c4575d3624aa3fc81188f0d9dd","observation_id":"46d8f48c-8d10-42b9-919e-3086c271500e","resolution":{"observed_at":"2026-08-07T21:56:21.267410Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.277593Z","title":"Intercode: Stan- dardizing and benchmarking interactive coding with execution feedback","venue":null,"work_id":"0869ecd1-811b-4097-8ba8-7d081a8071f1","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.409665Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:17d7513b5359524ff0603784b50d2f356de69da924ddc62c20482e0faae5530f","observation_id":"c04fb848-324c-4f81-8992-9abd600582b4","resolution":{"observed_at":"2026-08-07T21:56:21.282470Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.233819Z","title":"Grounded copilot: How programmers interact with code-generating models","venue":null,"work_id":"43ebcfc8-e5a4-4eae-a191-8e5e2d28c3f4","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.424972Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:aca2daacd2c475d58ad7991fe43c0f7aa65da0404e065611c93131285a63a84d","observation_id":"73350f3a-8230-4851-b9b8-997f8347744a","resolution":{"observed_at":"2026-08-07T21:56:21.239096Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.248798Z","title":"Bernstein, and Percy Liang","venue":null,"work_id":"cf98b337-846a-4162-936b-96036f9977f5","year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.419351Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:81da62935fe3b3572cb59fe45d4684a70882bda71876def8ab0955c07b107fc0","observation_id":"540a8e5f-e9f5-443e-8ff2-e0bf5aaf39ad","resolution":{"observed_at":"2026-08-07T21:56:21.253252Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.204921Z","title":"Ai-assisted code authoring at scale: Fine-tuning, deploying, and mixed methods evaluation","venue":null,"work_id":"d6d7986f-0085-4c3a-af9b-70753645d67b","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.435058Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:3a219c42145aea2e50a643fa993e26f4f61a5886da0a202a886adf6bf1a6d10b","observation_id":"a8cfb10c-ef32-4ff7-8e29-ddb62d0b92f4","resolution":{"observed_at":"2026-08-07T21:56:21.209861Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.219187Z","title":"Reading between the lines: Modeling user behavior and costs in ai-assisted programming","venue":null,"work_id":"296601b1-ee67-4e62-813f-a8f3092ea1c6","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.429404Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:2c350945c11743f099c0987e8200b2f8cf3c6ad5cdcdc1e3e2f3b9e11ebd3288","observation_id":"71b5cf91-4e19-456e-a9bb-d7c98a3a2b4a","resolution":{"observed_at":"2026-08-07T21:56:21.223990Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.190067Z","title":"The productivity effects of generative ai: Evidence from a field experiment with github copilot","venue":null,"work_id":"0be4c49f-2ac8-4422-abc2-e726de404669","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.444550Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:48d67ff2e0154ad746043a7de4d32340af5b374ccffa90e338b9050faa5efcc1","observation_id":"0e859cd5-d782-42cd-a18b-1f2bb70dc091","resolution":{"observed_at":"2026-08-07T21:56:21.194986Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.04596","last_updated":"2025-02-28T15:07:29Z","snapshot_observed_at":"2026-08-08T19:13:26.973455Z","submitted_at":"2024-10-06T19:11:55Z","title":"Need Help? Designing Proactive AI Assistants for Programming","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.04596","snapshot_observed_at":"2026-08-07T21:56:20.439848Z","title":"Need help? designing proactive ai assistants for programming","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.439848Z"},"links":{"cited_paper":"/paper/2410.04596","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:5ee881f246f4d4b09d5446e7b1af6612e9a7888e9e35d6e6952185ec2d8f6eba","observation_id":"ccc36f92-77ba-42e7-ab8e-2eb8904eee4f","resolution":{"observed_at":"2026-08-07T21:56:20.439848Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.14196","last_updated":"2024-01-26T09:23:11Z","snapshot_observed_at":"2026-08-06T22:40:28.707813Z","submitted_at":"2024-01-25T14:17:53Z","title":"DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.14196","snapshot_observed_at":"2026-08-07T21:56:20.454878Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.454878Z"},"links":{"cited_paper":"/paper/2401.14196","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:e9b89865301e564c5ffefe53b41f77b3cb6231f233745d2d825d32dfdee5e95f","observation_id":"31933b58-dde6-4e68-8a3c-cf9bf601c591","resolution":{"observed_at":"2026-08-07T21:56:20.454878Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.16197","last_updated":"2024-02-25T20:43:55Z","snapshot_observed_at":"2026-08-06T04:51:18.017441Z","submitted_at":"2024-02-25T20:43:55Z","title":"Language Models for Code Completion: A Practical Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.16197","snapshot_observed_at":"2026-08-07T21:56:20.449996Z","title":"Language models for code completion: A practical evaluation, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.449996Z"},"links":{"cited_paper":"/paper/2402.16197","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:082e7aa31cc45ad9afbff22e80cb116f24d14aca92decce5a0f6a54e3093e779","observation_id":"2b9efc6b-d5d2-4904-884f-2b24fcda3f97","resolution":{"observed_at":"2026-08-07T21:56:20.449996Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03716","last_updated":"2024-07-10T23:15:49Z","snapshot_observed_at":"2026-08-03T14:09:43.107865Z","submitted_at":"2023-10-05T17:38:28Z","title":"A Long Way to Go: Investigating Length Correlations in RLHF","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03716","snapshot_observed_at":"2026-08-07T21:56:20.465030Z","title":"A long way to go: Investi- gating length correlations in rlhf","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.465030Z"},"links":{"cited_paper":"/paper/2310.03716","citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:f4ae5e14302902c8812df344d8e59f8428a39841b31297d6595e3a2b928d08c5","observation_id":"b5e176e0-f468-4289-ba89-d9ffca8f0654","resolution":{"observed_at":"2026-08-07T21:56:20.465030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.174575Z","title":null,"venue":null,"work_id":"d9e4eef8-9179-46c1-a863-c1e3e29f89b8","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.460514Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:04a8565bc4d6174f85cc67e7955fbdbd90f7ecc3cc9851167829a287d83acc37","observation_id":"f7022013-a03c-4689-89cb-c5c3b4b5b5c0","resolution":{"observed_at":"2026-08-07T21:56:21.179604Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.158891Z","title":"PSM presents the code context in the order of prefix and then suffix, using XML notation to demarcate prefix, suffix, and middle segments (e.g., <PREFIX> and </PREFIX>)","venue":null,"work_id":"7387f120-1f9b-4506-bb94-94357453ec82","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.470711Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:4d4017d8e8b18c32780f78b94d16c782881d49409a5d946ba3dbc57cb3856530","observation_id":"bb660929-4a3a-47fd-a5df-dbd61cad2744","resolution":{"observed_at":"2026-08-07T21:56:21.164432Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.142441Z","title":"SPM is identical to PSM except that the suffix appears before the prefix, which may be more natural than having the suffix appear directly before the output as is the case with PSM","venue":null,"work_id":"742ff606-c966-4b44-a16c-59d9f8031660","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.475346Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:52c71b823fad6f44c485eaf71615e1f20bd6608c6a73bdea816024b377b407db","observation_id":"791a6138-a9d7-41d7-bdd7-cce2f8ebc414","resolution":{"observed_at":"2026-08-07T21:56:21.148917Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.125748Z","title":"sentinel","venue":null,"work_id":"b2442737-3698-4ee0-98b8-db01e5c16f35","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.480939Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:c44ed8b9d74f1ee7e968238272dd98aaf32c9f553f4b8e528c64eeda82424d62","observation_id":"59563f00-3eb9-4531-a2ea-6321800fa2af","resolution":{"observed_at":"2026-08-07T21:56:21.130744Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.109948Z","title":"pre-fill","venue":null,"work_id":"121e7e4c-1617-4edf-b1ca-a93cca7cd1ca","year":2024},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.485696Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:dbb3b0d5418a994c5fdac3703a19a595eef641a01a98765fb81a96e79f1f075d","observation_id":"9c32a364-eb9a-4742-b1de-23f3c3bfc799","resolution":{"observed_at":"2026-08-07T21:56:21.114665Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.094627Z","title":null,"venue":null,"work_id":"03d29335-fc33-497c-8adb-e027f5ad7f0d","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.490889Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:7f301aaa531dc89c891391a67e0075d6dc807bf77f3154167ad08128c7e751f6","observation_id":"c358e7f6-d056-4f08-888d-1fc8afcec11a","resolution":{"observed_at":"2026-08-07T21:56:21.099436Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.079803Z","title":null,"venue":null,"work_id":"d25aa941-a879-4e8c-9eb4-bf105056f1da","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.495527Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:defc325e9060779e6cee03b3ef4a154dc868801f056e569eb2e41bc4bea31906","observation_id":"1ef2a7c9-b9d0-44fb-963c-75795b01b002","resolution":{"observed_at":"2026-08-07T21:56:21.084359Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.063343Z","title":null,"venue":null,"work_id":"eeaae772-3fce-4904-91a7-4a6a453b5aa9","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.501488Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:eb175c7c212ce59663d85de5644785461cfe708dfd3c48b1128ab0211ed2c078","observation_id":"2f161a4e-60b7-4ac0-a093-1ffe17e6017e","resolution":{"observed_at":"2026-08-07T21:56:21.068356Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.046696Z","title":"clusters","venue":null,"work_id":"cf5ce956-10eb-4447-b36a-e3fc1dabff12","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.506734Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:e1b30753b624fbdcdefe6a7b6bfefc2fa06f9c400059a1bb6c31a41d8846fb42","observation_id":"f9adbe6a-8e84-4cf6-b88d-b811ccd129e9","resolution":{"observed_at":"2026-08-07T21:56:21.052539Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T21:56:21.559332Z","title":null,"venue":null,"work_id":"0a2729b5-7e64-4dba-8e6c-82a4c6b826b0","year":null},"citing_paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-07T21:56:20.204693Z"},"links":{"citing_paper":"/paper/2502.09328"},"observation_digest":"sha256:6779de86a4031828d626dc5f497cd42d9bae7866e1fce3e0837f4be78fb3f59a","observation_id":"8895e605-430e-4142-b4ee-c77c5f9b66b9","resolution":{"observed_at":"2026-08-07T21:56:21.564333Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2502.09328","last_updated":"2025-02-13T13:40:52Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-07T21:49:22.645033Z","submitted_at":"2025-02-13T13:40:52Z","title":"Copilot Arena: A Platform for Code LLM Evaluation in the Wild"},"reference_resolution":{"displayed":67,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":38,"verified_exact":0,"verified_fuzzy":29},"total_outbound_references":67},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 67 of 67 outbound references and 5 inbound Pith citation observations for arXiv:2502.09328."}