{"as_of":"2026-08-19T04:35:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:3badf3ed95b673a262b2716d02f9e61625aed0c933dbf49879ff589c166652c6","coverage":[{"denominator":61,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":61,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T20:41:09.087338Z","state":"measured"},{"denominator":61,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":61,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-18T06:34:40.430872+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2505.12331/citation-record","integrity":"/paper/2505.12331/integrity","json":"/paper/2505.12331/citation-record.json","paper":"/paper/2505.12331"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.762407Z","title":"Github copilot.https://copilot.github.com, 2021","venue":null,"work_id":"d9ee8c2a-d932-4158-ae9d-a086a09a7a74","year":2021},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.882885Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:8cf34dc7e453c8d2344437694de6dc3ab23561aaf5d0991b085bad5062960945","observation_id":"421c77fd-dd77-4d41-8c94-8255de3985fa","resolution":{"observed_at":"2026-08-15T20:41:09.765896Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.752463Z","title":"Cursor: The ai-powered code editor.https://cursor.so, 2023","venue":null,"work_id":"493fcd1d-fa03-4f3a-a335-a31867d35b7e","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.887126Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:699f4480a9315e246c9e7b471f9d1d3b48e5341237c8925beeed45bad5de7966","observation_id":"9420a2ab-6a59-4c0a-ad9c-199c6e3e0c0d","resolution":{"observed_at":"2026-08-15T20:41:09.755646Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.890792Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.890792Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:e652a2b32807a436f872f18bb4949292ba1a788060c1dd4f7ef7f199dee0b713","observation_id":"37a4dd52-72cb-4904-9535-2acb97562cc9","resolution":{"observed_at":"2026-08-15T20:41:08.890792Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.734430Z","title":"Codelmsec benchmark: Systematically evaluating and finding security vulnerabilities in black-box code language models","venue":null,"work_id":"8d70c3b4-cbe2-42c3-b297-3fe7bcf29fd2","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.894506Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d381d388aeac7bea906e9e89c1df2565fb8a5e78272adb8dd8d27a4de6f30135","observation_id":"ffd97942-0d28-4e3f-8692-fa5f96fc972c","resolution":{"observed_at":"2026-08-15T20:41:09.738087Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2503.10452","last_updated":"2025-05-29T13:17:33Z","snapshot_observed_at":"2026-08-16T12:50:20.510618Z","submitted_at":"2025-03-13T15:18:56Z","title":"DynaCode: A Dynamic Complexity-Aware Code Benchmark for Evaluating Large Language Models in Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.10452","snapshot_observed_at":"2026-08-15T20:41:08.898224Z","title":"Dynacode: A dynamic complexity-aware code benchmark for evaluating large language models in code generation.arXiv preprint arXiv:2503.10452, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.898224Z"},"links":{"cited_paper":"/paper/2503.10452","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:3e69b705f5a015a4da2fdd38c82589c99a257bab0ed3a956add52933aac994fa","observation_id":"f4362cce-2d41-45cc-9720-d8e55ab24d87","resolution":{"observed_at":"2026-08-15T20:41:08.898224Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.724381Z","title":"Humanevo: An evolution-aware benchmark for more realistic evaluation of repository-level code generation","venue":null,"work_id":"ecf143db-548b-453e-ba50-e57aa5439d62","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.902726Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0c52d9ea4c803187a4256999b7507c2207f7c30f0d75725891f9e1f3305625fd","observation_id":"92931cb0-3c53-4536-bab2-27d63c5c177c","resolution":{"observed_at":"2026-08-15T20:41:09.728038Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.906493Z","title":"Codeif-bench: Evaluating instruction-following capabilities of large language models in interactive code generation.arXiv preprint arXiv:2503.22688, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.906493Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:87f5df045d255e11a39dc004e74b52567294e3c3c147e0f0cf12cf50a62bd63b","observation_id":"be0cf7b6-15bc-4966-9283-e7b95304764b","resolution":{"observed_at":"2026-08-15T20:41:08.906493Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-15T20:41:08.909862Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code.arXiv preprint arXiv:2311.09835, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.909862Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:723ff996bfb6f40e90cd5c5fab44f0d584fd56a4bf687d4f3bd947927876adaf","observation_id":"19ebd076-1d70-427d-912f-77193fece6a3","resolution":{"observed_at":"2026-08-15T20:41:08.909862Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-08-18T08:11:45.716032Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-15T20:41:08.913882Z","title":"Swe-bench: Can language models resolve real-world github issues?arXiv preprint arXiv:2310.06770, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.913882Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d3d8895add91afac0f342afb5bd2e54bfa1e42ecf4f1fe6645afae0c4a35d58f","observation_id":"0fcf29d9-48f7-44a6-bf52-a8bd0d4fd83b","resolution":{"observed_at":"2026-08-15T20:41:08.913882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.714760Z","title":"Wang, Armando Solar-Lezama, Koushik Sen, and Ion Stoica","venue":null,"work_id":"56779f09-816c-42f6-a73d-4d18235b55c4","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.917414Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:fbb1d609aace80c86a44dd87da743751b3d47b4f39d019435cb2bd9cd012e33b","observation_id":"be14a1d2-d442-401c-a987-89953bee9903","resolution":{"observed_at":"2026-08-15T20:41:09.718194Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.704694Z","title":null,"venue":null,"work_id":"ae9f4251-bd21-42e6-a852-043430058e04","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.920715Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:49afa9caa76734824d34d817d168fcb97b9de42c052e97d89a208f3d7d5b043e","observation_id":"1f926510-652a-45e2-81e3-6297ad342f41","resolution":{"observed_at":"2026-08-15T20:41:09.708228Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2504.02605","last_updated":"2025-04-03T14:06:17Z","snapshot_observed_at":"2026-08-09T23:51:45.534260Z","submitted_at":"2025-04-03T14:06:17Z","title":"Multi-SWE-bench: A Multilingual Benchmark for Issue Resolving","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.02605","snapshot_observed_at":"2026-08-15T20:41:08.924377Z","title":"Multi-swe-bench: A multilingual benchmark for issue resolving, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.924377Z"},"links":{"cited_paper":"/paper/2504.02605","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:ee0bd1036a954fff5f79fd38ad9b0facb6ac4a4029fd870c637b4939d38d483b","observation_id":"5aabb163-a1e4-4c76-8171-de64910dd20d","resolution":{"observed_at":"2026-08-15T20:41:08.924377Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.694479Z","title":"SecRepoBench: Benchmarking LLMs for secure code generation in real-world repositories, 2025","venue":null,"work_id":"fb072d00-d155-4014-ac3a-872dd26869cc","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.928133Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a32508b16e0bde604cda516b6578426c89f591650b19c0f4c7e8ec2d2a3099bf","observation_id":"4f899914-f0ea-4ffe-be68-ef5008893aec","resolution":{"observed_at":"2026-08-15T20:41:09.698118Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.684247Z","title":"CWEval: Outcome-driven evaluation on functionality and security of LLM code generation, 2025","venue":null,"work_id":"a23f801e-20c3-463b-9a3a-f4ea04357117","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.931450Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:635d21195c32e073700ed402549e77d95709561d267edaa438421f6f356d1f10","observation_id":"175ab0dc-7966-411f-b6c0-7ddeb40f37a1","resolution":{"observed_at":"2026-08-15T20:41:09.687983Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.11470","last_updated":"2024-10-09T05:59:07Z","snapshot_observed_at":"2026-08-16T13:33:52.529038Z","submitted_at":"2024-07-16T08:08:48Z","title":"Beyond Correctness: Benchmarking Multi-dimensional Code Generation for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.11470","snapshot_observed_at":"2026-08-15T20:41:08.934788Z","title":"Beyond correctness: Benchmarking multi-dimensional code generation for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.934788Z"},"links":{"cited_paper":"/paper/2407.11470","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:b392067a813c9902da4cf3da55336a0be28a7e07fcfcdf549c9bbd31511dd8fa","observation_id":"d267b0db-c1a3-4ef8-b916-cecb179222a6","resolution":{"observed_at":"2026-08-15T20:41:08.934788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.674025Z","title":"Codearena: Inspecting and improving code quality metrics using minecraft","venue":null,"work_id":"60876434-aa41-4db0-b8f5-69fcb1ef271e","year":2019},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.938516Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:54e8cb4efdc1a37e1056884db1a18d8390f60c5d717977d15575f4deb02efeca","observation_id":"8097f06e-e716-4baa-a9f5-f247dd6ddd16","resolution":{"observed_at":"2026-08-15T20:41:09.677609Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.663530Z","title":"CodeElo: Benchmarking competition-level code generation of LLMs with human-comparable elo ratings, 2025","venue":null,"work_id":"dc772053-e01f-4e3c-8984-eb604bbe1fe1","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.942026Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:ede0b3e2033c2e9dcacffb9d76cef6cd58050e992932d77afb1a6f9e4e579982","observation_id":"838d148b-e976-4a6a-8eae-01547b8c5a40","resolution":{"observed_at":"2026-08-15T20:41:09.667110Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.652858Z","title":"ComplexCodeEval: A benchmark for evaluating large code models on more complex code","venue":null,"work_id":"08e6eb4d-6486-4d42-8fc3-c0c27cf6073f","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.945410Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:47d049e57f275bfea449c3dcc6b2a70eb5233e554f10b06c7664db3fee93637c","observation_id":"f7e97651-5964-4935-a92f-b6670019008f","resolution":{"observed_at":"2026-08-15T20:41:09.656575Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.642535Z","title":"PythonSaga: Redefining the benchmark for code generating LLMs, 2024","venue":null,"work_id":"8200056c-7fd7-4053-8d63-50c23328bd8c","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.948865Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:75601937052fe0619aca0da3788e1bc11cba211d52ce12f437d23ff6674ba63f","observation_id":"d54edff5-10a6-4393-a1e0-24f00abd278f","resolution":{"observed_at":"2026-08-15T20:41:09.646182Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.18573","last_updated":"2025-03-17T17:58:13Z","snapshot_observed_at":"2026-08-16T12:59:57.442596Z","submitted_at":"2024-12-24T17:56:08Z","title":"Top General Performance = Top Domain Performance? DomainCodeBench: A Multi-domain Code Generation Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.18573","snapshot_observed_at":"2026-08-15T20:41:08.952533Z","title":"How well do llms generate code for different application domains? benchmark and evaluation.arXiv preprint arXiv:2412.18573, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.952533Z"},"links":{"cited_paper":"/paper/2412.18573","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a98bffc7258af1f87f2e04102ddf60eb167e228b014e39c3a6a44f4ac46ca15c","observation_id":"b56dff38-2269-4311-9d09-2d1988dba53b","resolution":{"observed_at":"2026-08-15T20:41:08.952533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.631711Z","title":"Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik Narasimhan","venue":null,"work_id":"2231f6ef-4dee-441f-991d-23183411ecd3","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.956417Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:56a10e9625ee159a2d610e323a721b93b48219c68d83163605892b431843d580","observation_id":"5d8a4b47-36f1-4ff6-8f67-cc13055f3cd5","resolution":{"observed_at":"2026-08-15T20:41:09.635334Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.620498Z","title":"ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation, 2023","venue":null,"work_id":"cd58e8e5-98b8-4b48-9b3b-af1d46d698d5","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.959822Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:1df4c8ccb9a76b6a35be374931b0982688004834335911cf2342d0076262b3fc","observation_id":"f99279ee-a255-4536-9367-6778d381a976","resolution":{"observed_at":"2026-08-15T20:41:09.625092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.609497Z","title":"HumanEval-XL: A multilingual code generation benchmark for cross-lingual natural language generalization, 2024","venue":null,"work_id":"5eccf92e-9334-4f37-aa56-78ec808f7f35","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.963333Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:1256b78a5cc27989a728352f57b1a433d41e63fc73f1ff48ad0e10512b8d2e95","observation_id":"2cbeb47d-f80b-4c4f-86fb-534d76cc5f87","resolution":{"observed_at":"2026-08-15T20:41:09.613243Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.966684Z","title":"Codegeex: A pre-trained model for code generation with multilingual benchmarking on humaneval-x","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.966684Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:46db9646d5d2be499c15877f6c49a5392b159e7d39afc8fde40c6e1b6b348374","observation_id":"996c0b14-7950-4375-9405-0055fdad5d9e","resolution":{"observed_at":"2026-08-15T20:41:08.966684Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.970147Z","title":"{AddressSanitizer}: A fast address sanity checker","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.970147Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4b93a34c62c44514dae54a236e3bed89398b2b6f84cd33a36cad346c49caafe3","observation_id":"25f43123-634e-4ac1-998e-991732e8da59","resolution":{"observed_at":"2026-08-15T20:41:08.970147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.593210Z","title":"php-src: The php interpreter.https://github.com/php/php-src, 2025","venue":null,"work_id":"7ce77d0c-dee8-4143-8265-60fd47c0a3da","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.973421Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a3530286d194a4e597ea8c9f16005067536ab7f18e053749498fbef9c468e8b1","observation_id":"74e819d9-d3f8-47ba-9843-c946e1a8f32b","resolution":{"observed_at":"2026-08-15T20:41:09.596794Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.582932Z","title":"Richard Hipp","venue":null,"work_id":"7851ace3-f84d-4a00-9a13-93adee6c61fb","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.976672Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4eb8ee91e74e464c1adf5c327012c076a5c79c8e90e04d4e62e91b99fb3391a9","observation_id":"b09990d6-ebf5-434e-9d9b-0c5d93593900","resolution":{"observed_at":"2026-08-15T20:41:09.586385Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.571157Z","title":"https://testing.googleblog.com/2020/08/ code-coverage-best-practices.html, 2020","venue":null,"work_id":"75ad66af-8a97-4cc3-8618-932afe53922a","year":2020},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.979969Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:92541a180809a826644a6263148a83fc9cfdb8fc59df7298721c81f4b35d2bbf","observation_id":"6e377465-2058-4c79-ac59-b801173648b5","resolution":{"observed_at":"2026-08-15T20:41:09.574840Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.560897Z","title":"libclang: C interface to the clang library","venue":null,"work_id":"6a81702e-ba39-417d-99ed-34509f164e42","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.983338Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:1aebb12bbe338da054b87406a94931c7e1c072e7269c54ea1daf24f324e2d6b2","observation_id":"88dc70a5-32bc-440f-abeb-a04a7fac21bc","resolution":{"observed_at":"2026-08-15T20:41:09.564370Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.987369Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.987369Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d572b9208a601ad75d46ac4cdd79ceb8222e46f812b50866c16c1a0b5966f72e","observation_id":"d1450324-a4f6-45df-a943-c1b8d6b4b472","resolution":{"observed_at":"2026-08-15T20:41:08.987369Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.544190Z","title":"difflib — helpers for computing deltas between objects","venue":null,"work_id":"71acb22c-2ee3-4ab9-b0de-d6aab1f6118b","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.990624Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:3e56399854f7878fbff842eaef0677c86c8f46501b874840d58700de9c11b5e0","observation_id":"1978e6ba-1cb1-4ecc-bc52-25c65abf6ff3","resolution":{"observed_at":"2026-08-15T20:41:09.548044Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.534683Z","title":"gpt-o1 model.https://platform.openai.com/docs/models/o1, 2025","venue":null,"work_id":"4ed32e90-4dfa-491e-aa1a-7e1954960f95","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.993964Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:afb84d758da9db697a3aaca6fc732781d9b0ebf560666cdc4a85a30dae04883b","observation_id":"cd2d49f0-5dc8-4fae-98e3-42c38be67156","resolution":{"observed_at":"2026-08-15T20:41:09.538208Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.525080Z","title":"o3-mini model.https://platform.openai.com/docs/models/o3-mini, 2025","venue":null,"work_id":"02796f5c-9207-4c38-807f-fd8b97af4839","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.997232Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:7095befafb12ad18e3a3daea9a936c28995abbfee514470abc3a91b31a5dcab8","observation_id":"ab3f1842-84bd-444f-b18d-d960ecb00322","resolution":{"observed_at":"2026-08-15T20:41:09.528623Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.515439Z","title":"Claude 3.7 sonnet.https://www.anthropic.com/claude/sonnet, 2025","venue":null,"work_id":"aa9e2f17-1f7c-403c-83c8-af02e399ab0f","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.000387Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0c4a16894d1e3d7eeef16686188d43853c91606fdaf57765959434002851ff59","observation_id":"c5d584d7-b454-482d-bd96-1006e253fffb","resolution":{"observed_at":"2026-08-15T20:41:09.518845Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.505879Z","title":"Claude 3.5 haiku.https://www.anthropic.com/claude/haiku, 2025","venue":null,"work_id":"7561c8a6-2043-4fe9-9549-825687e7ef92","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.003876Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:eeba262ea77f63204b30161463ee01591fddb683aacc098bc24745a6a037fc27","observation_id":"5ac39b2b-67e2-42b1-a827-d6712bfb99da","resolution":{"observed_at":"2026-08-15T20:41:09.509216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.495872Z","title":"Gemini 2.5 flash.https://developers.generativelanguage.google/, 2025","venue":null,"work_id":"e0262a86-601a-45e4-9947-8fe5fae39e69","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.007301Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d69a76e9b345204f3b8b458171781be1917cbd73bccedd31f02ed7d9c1498a8d","observation_id":"ecb39fce-e3dc-4376-b46c-2dc6b95967f7","resolution":{"observed_at":"2026-08-15T20:41:09.499364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.486152Z","title":"Llama 3.3 70b instruct (fp16)","venue":null,"work_id":"a2ae3316-da58-4e0e-8e00-f98543aef075","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.010502Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:8eb053b84a0fb4823dd1aaee094729ba13167491348d0fa9952bfef338cc2791","observation_id":"614a4c67-f17b-462e-afc7-18d9722d4bb9","resolution":{"observed_at":"2026-08-15T20:41:09.489480Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.476608Z","title":"Codellama 70b instruct (fp16)","venue":null,"work_id":"13b9c6cb-465f-4bb3-9e5a-a4ff8fe236b6","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.013703Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:18e280622450cc2e73ad834f9eb7c92d6fd1068284378d4730eea3a977c67c48","observation_id":"3e19ca2f-c7eb-4839-9edb-39d90a5268ed","resolution":{"observed_at":"2026-08-15T20:41:09.479853Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.466903Z","title":"Qwen 2.5 coder 32b instruct (fp16)","venue":null,"work_id":"40e678b2-3366-4413-91fb-f0259d6ff7b5","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.016914Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:8be1d60c517e8690bc72f088b0a342066ec7644bad0fec98a38b2f1cf385d1d7","observation_id":"d97379d9-4aec-40a6-af08-2b98a80f12bc","resolution":{"observed_at":"2026-08-15T20:41:09.470476Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.456834Z","title":"Qwen 3.0 30b-a3b fp16.https://ollama.com/library/qwen3:30b-a3b-fp16, 2025","venue":null,"work_id":"11470e59-eb75-4ab9-9862-6ec3581c4be6","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.020235Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:69aaaa9dca4bdaf3af24ec7b7a72b5bb4b67aa9da9be5c427a26d6f096f6dbe4","observation_id":"16c0acef-3449-42ba-8439-7306efa9a235","resolution":{"observed_at":"2026-08-15T20:41:09.460304Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.447220Z","title":"Qwen 3 8b fp16.https://ollama.com/library/qwen3:8b-fp16, 2025","venue":null,"work_id":"9dc3ec90-0621-47ed-a55b-d86741a8177f","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.023486Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:97286da36b67fafed44d981578037b4f60747e90ff765d1de48d7ebdb459b467","observation_id":"36fdb130-8cbe-47e1-ad88-2d20377819a9","resolution":{"observed_at":"2026-08-15T20:41:09.450714Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.437295Z","title":"Gemma 3 27b-it fp16.https://ollama.com/library/gemma3:27b-it-fp16, 2025","venue":null,"work_id":"731b3efe-4114-473b-b160-334c019eecf4","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.026695Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:e62fbc30a9ebfe56d7324ff5828ccd384627199e4dcc4be8d823f3c627be3f6b","observation_id":"ed415a7d-841a-4574-9f59-df739406e666","resolution":{"observed_at":"2026-08-15T20:41:09.440770Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.427673Z","title":"Qwen 2.5 coder 14b instruct (fp16)","venue":null,"work_id":"dc764a59-fc37-46ec-b2b1-997c675ce47d","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.029928Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:b8476b154c679bec8a759f512d5defa84885641c7ad5319ba8103a09983bbfaf","observation_id":"f14f87fd-c5ed-4fa4-9171-ec76af6b46ed","resolution":{"observed_at":"2026-08-15T20:41:09.430956Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.418038Z","title":"Deepseek coder v2 16b lite instruct (fp16)","venue":null,"work_id":"d4368407-6fc7-45ec-91f1-6e01cb845677","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.033335Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:094c2242f3618cc5258b6be2518fbed8dc2a0958f667dccef11203601ced5a47","observation_id":"2e09b457-c428-4822-8ef3-01b67b7c49c2","resolution":{"observed_at":"2026-08-15T20:41:09.421381Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.408557Z","title":"Starcoder2-15b-instruct-v0.1 (fp16)","venue":null,"work_id":"d08ebbe6-0a08-41e1-96a2-4a9e447ea5de","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.036683Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4940f6426bb00ace8e37437f3b940f64e2523c847b980eeffc706496fd4e10a6","observation_id":"2651a8ef-fb93-47ae-a2dc-546876a5fc10","resolution":{"observed_at":"2026-08-15T20:41:09.411806Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.399435Z","title":"Phi-4 14b fp16.https://ollama.com/library/phi4:14b-fp16, 2025","venue":null,"work_id":"8d8a273c-4f94-470f-8a40-0d8cf119e0fd","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.039967Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0f1bb336c3faf4997fb014967a29cbcf6022487f6534b174e11def47acbe53ac","observation_id":"065a418e-8f61-4bad-9e28-8c86c4b37fce","resolution":{"observed_at":"2026-08-15T20:41:09.402729Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.389546Z","title":"Mistral 7b instruct (fp16)","venue":null,"work_id":"374430d6-0a45-4822-bef5-f8131a54bbfd","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.043072Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:77530c9c5f4e2d709356872a0485f1c1a760c8e065d9c0724feb11cf9ec56094","observation_id":"e8a8fc19-035d-41bc-9fb8-27aa8ccfdf02","resolution":{"observed_at":"2026-08-15T20:41:09.392942Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.379087Z","title":"Codegemma 7b instruct (fp16)","venue":null,"work_id":"d3f285fb-af88-45e0-a7c8-fa4c60bbf277","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.045955Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:3947574990b6977ba0b53f724f8e538eb02d547855d03a5a0d77dba6a3f7fc20","observation_id":"56c62f9f-203a-46fc-bb32-8424f361b066","resolution":{"observed_at":"2026-08-15T20:41:09.382442Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.369252Z","title":"Openai: Advances in safe and beneficial ai.https://openai.com, 2025","venue":null,"work_id":"f5598d5f-504f-44c6-8e4a-fcec737a043c","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.048955Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f1511c27f3dcbe76dab974850f667229abe2b713b1e4482389654bf3e0c3c1c2","observation_id":"51c00e5f-a141-4348-b0c7-1541b8d6090b","resolution":{"observed_at":"2026-08-15T20:41:09.372657Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.358847Z","title":"Anthropic: Building reliable, steerable ai systems","venue":null,"work_id":"92d76da9-2e32-41c8-a171-373337a9d36b","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.052241Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f20e455a11d7e3544f9d59284a74f14619fdd83741c8a1367f959697e16ace69","observation_id":"1074e2ff-727f-496b-800a-30dcd2e6b1b3","resolution":{"observed_at":"2026-08-15T20:41:09.362447Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.348857Z","title":"Google: Organizing the world’s information.https://www.google.com, 2025","venue":null,"work_id":"13dfaf51-b149-473f-9edb-615b58078fd9","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.055397Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:46a96eb91ad6907785f497babc8dacda4ea9784be8eaad982f9db47a665dfbe1","observation_id":"faa5be50-d86b-462e-97bc-7f4e4e873a70","resolution":{"observed_at":"2026-08-15T20:41:09.352265Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.338927Z","title":"Deepseek: Developer of high-performance open-source llms","venue":null,"work_id":"e9c32ac2-acc9-4600-9a90-cc12c38650d2","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.058832Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:2ecae03e5a1b6a6b08d01e69a90a71c45c68cff6da3ce1abc3ac5b73031a1009","observation_id":"b029a42d-0535-4919-b5d9-939351530fc5","resolution":{"observed_at":"2026-08-15T20:41:09.342614Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.328286Z","title":"Alibaba group: Global trade and technology.https://www.alibaba.com, 2025","venue":null,"work_id":"40a8c1cb-5b1f-49b1-ada6-2af688814875","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.061970Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d1652e87084cbef372107b52ea24eb6ec62492ef755fc5f1fbd59cbee8a6ce01","observation_id":"c12238fa-e33f-4420-ac49-bcf5605bd33e","resolution":{"observed_at":"2026-08-15T20:41:09.332122Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.317399Z","title":"Meta: Bringing the metaverse and social technology together","venue":null,"work_id":"c53bf76b-4a4c-4741-ba9b-e743306c13db","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.065031Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:151315f5f14bbf2d36295a35388c23b2e26a6ab8241d4dd15fd671111f0ef076","observation_id":"c9f9aa06-2519-4215-acfd-e96bd8cb4b54","resolution":{"observed_at":"2026-08-15T20:41:09.321033Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.306746Z","title":"Microsoft: Empowering every person and organization","venue":null,"work_id":"9d55bcc0-fe0d-45e9-a6ac-652173b0b644","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.068195Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d00f444bb038911970efb473e032e36a468a8079f4c64fbff1a066bfe85942d4","observation_id":"cded8149-9ca2-4f3e-80e8-5a24d16e8f85","resolution":{"observed_at":"2026-08-15T20:41:09.310513Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.296643Z","title":"Bigcode: Open and responsible development of code llms","venue":null,"work_id":"879e4e8b-f773-4500-8b29-b669bcbce97c","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.071519Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:60c431d3b9a52b2b705738603c103b819084a2551041ef014b022355bd109421","observation_id":"1ab6e7d1-79b9-4dc6-904f-a9092340bc36","resolution":{"observed_at":"2026-08-15T20:41:09.299952Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.286313Z","title":"Mistral ai: Frontier ai in your hands.https://mistral.ai, 2025","venue":null,"work_id":"01b91de0-eb65-49d5-9323-6b2b58eab765","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.074554Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a6074056cce7cfe1848365e0150256b6fedb3d12738f27d05c700d65d8a42b8d","observation_id":"2c5f0684-a7fa-4f47-ab91-988aacd0d958","resolution":{"observed_at":"2026-08-15T20:41:09.289957Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.275815Z","title":"Ollama: Get up and running with large language models locally","venue":null,"work_id":"3ac8d607-252b-4038-a5cc-9be9eed47ecf","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.077778Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:855386d13df290d376200542eb5eab6f673ae04597899289b3f985147e2447bd","observation_id":"77e66e68-6bcb-4ffd-a89b-8398e909a334","resolution":{"observed_at":"2026-08-15T20:41:09.279340Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1906.04908","last_updated":"2019-06-12T03:13:18Z","snapshot_observed_at":"2026-08-19T00:17:44.944869Z","submitted_at":"2019-06-12T03:13:18Z","title":"SPoC: Search-based Pseudocode to Code","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1906.04908","snapshot_observed_at":"2026-08-15T20:41:09.080649Z","title":"Spoc: Search-based pseudocode to code","venue":null,"work_id":null,"year":1906},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.080649Z"},"links":{"cited_paper":"/paper/1906.04908","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:62c7814588bed830141481b7f5142f38252dadee7f7e57113ea62d7be323c9b4","observation_id":"6376ec4b-63b3-4a79-bb6d-4e07616eb100","resolution":{"observed_at":"2026-08-15T20:41:09.080649Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T20:41:09.084092Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.084092Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:059e408add49683be7a965cda02476705c48e6d38be3eaa91e82b97927e8736a","observation_id":"c79ca41e-c891-4d6a-b74f-17d8593de675","resolution":{"observed_at":"2026-08-15T20:41:09.084092Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21713","last_updated":"2025-02-04T03:02:58Z","snapshot_observed_at":"2026-08-16T13:05:07.586558Z","submitted_at":"2024-10-29T03:54:59Z","title":"Fuzzing the PHP Interpreter via Dataflow Fusion","version":2},"cited_work":{"arxiv_id":"2410.21713","doi":null,"metadata_source":"pith","pith_arxiv_id":"2410.21713","snapshot_observed_at":"2026-08-15T20:41:09.123046Z","title":"Fuzzing the PHP Interpreter via Dataflow Fusion","venue":"cs.CR","work_id":"e98cfdd8-89d6-4000-8951-ad7fce986284","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.087338Z"},"links":{"cited_paper":"/paper/2410.21713","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:dbb15fbc6b435aa541e0427702fb5eb502e458a7abbfa6382231f1b256a03e57","observation_id":"ca45235d-09f5-492e-9102-3bbbf33dedfd","resolution":{"observed_at":"2026-08-15T20:41:09.129077Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","latest_version":2,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs"},"reference_resolution":{"displayed":61,"state_counts":{"malformed_identifier":0,"metadata_mismatch":1,"parse_uncertain":0,"unresolved":14,"verified_exact":0,"verified_fuzzy":46},"total_outbound_references":61},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 61 of 61 outbound references and 0 inbound Pith citation observations for arXiv:2505.12331."}