{"as_of":"2026-08-05T00:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:98a18d98048fac5c358d86974115e72e4676a07d90ac37569bf3e90df1d08f80","coverage":[{"denominator":11,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":11,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T23:43:15.246548Z","state":"measured"},{"denominator":12,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":12,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-04T06:34:03.388597+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-28T07:16:30.725358Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-02T06:56:44.727065Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"cited_work":{"arxiv_id":"2511.04355","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2511.04355","snapshot_observed_at":"2026-07-07T03:17:14.138901Z","title":"M., Heydari, M., Kazerooni, P., Maninger, D., and Mezini, M","venue":null,"work_id":"b49192ec-c1d4-4300-a810-5dd7f4758afe","year":null},"citing_paper":{"arxiv_id":"2606.09878","last_updated":"2026-06-03T01:28:00Z","snapshot_observed_at":"2026-07-06T23:49:08.412079Z","submitted_at":"2026-06-03T01:28:00Z","title":"FailureScope: Cross-Regime Behavioral Diagnosis of Language Model Weaknesses","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-06-28T07:16:30.725358Z"},"links":{"cited_paper":"/paper/2511.04355","citing_paper":"/paper/2606.09878"},"observation_digest":"sha256:372a09dd80b6e5fdc127541d1c4fc7c13b3e2380aa7e045b036de22f4c458cc1","observation_id":"a62c1ff0-61b9-4a91-91ea-ba6bd33f0908","resolution":{"observed_at":"2026-07-07T03:17:14.138901Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-04T06:34:03.388597+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-04T06:34:03.388597+00:00","source":"crossref"},{"observed_at":"2026-08-04T06:33:57.428241+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2511.04355/citation-record","integrity":"/paper/2511.04355/integrity","json":"/paper/2511.04355/citation-record.json","paper":"/paper/2511.04355"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:14.703226Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.703226Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:f65da3833639becde98f76fe31d9accdf66b71b84aa2f6d38d1db65e43934fa0","observation_id":"cad37e20-552c-414a-859e-82199c07ed36","resolution":{"observed_at":"2026-08-03T23:43:14.703226Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:14.735306Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.735306Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:360fcbc684e3c1c38774098a46984c45164c219a1385d8b61fbd4fae693f11fa","observation_id":"b212e767-b003-45ec-9d31-97f35e21915d","resolution":{"observed_at":"2026-08-03T23:43:14.735306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-03T23:43:14.786547Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.786547Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:85332b6fae8b69ac290315cf0ee07093b7e78b917dc2762caa81d33c31317077","observation_id":"d89f27f7-d5a2-4b29-b2a9-4479aab96e78","resolution":{"observed_at":"2026-08-03T23:43:14.786547Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-03T23:43:14.853124Z","title":"Austin, A","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.853124Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:1333bcee695b26e19ed3f1ec2bc2537fa61b549c9cf25bdbd5590a8462a599c1","observation_id":"a8f4ff59-3277-4774-950d-ec6952061c44","resolution":{"observed_at":"2026-08-03T23:43:14.853124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.00699","last_updated":"2025-07-09T19:56:02Z","snapshot_observed_at":"2026-07-06T17:53:40.016965Z","submitted_at":"2024-03-31T14:32:02Z","title":"A Comprehensive Survey of Contamination Detection Methods in Large Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.00699","snapshot_observed_at":"2026-08-03T23:43:14.882558Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.882558Z"},"links":{"cited_paper":"/paper/2404.00699","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:e27d5fcdf8300a00c2ec4e0e709b7b97554f7030cf8768f4831a140d5c3d5aa6","observation_id":"b511eb66-97da-4e5a-9526-7986e81dd6df","resolution":{"observed_at":"2026-08-03T23:43:14.882558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-07-31T19:00:59.311189Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-03T23:43:14.940179Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.940179Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:8d797ac37193c8354448cb55d535ec9da257bf9d9ffa956b4ea511a0bbfa21d4","observation_id":"0f02549c-e2cb-47f2-ad33-7dd4011b3610","resolution":{"observed_at":"2026-08-03T23:43:14.940179Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-03T23:43:14.983161Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.983161Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:c840949eca689b2f1cec411b657bc547de1fe5afe5a7d0036d97119569f87751","observation_id":"56be89f8-b821-4f69-b6f4-d5f5a7191cdb","resolution":{"observed_at":"2026-08-03T23:43:14.983161Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:15.074745Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.074745Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:90380c79adb3ffae1bbff0cfc802c401ede190ba5c01ec377f505caf95ea8d93","observation_id":"e4ee9963-cdfa-4185-a2a3-c30eee9424e3","resolution":{"observed_at":"2026-08-03T23:43:15.074745Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.21199","last_updated":"2024-12-31T08:20:42Z","snapshot_observed_at":"2026-07-06T20:14:49.976782Z","submitted_at":"2024-12-30T18:58:58Z","title":"HumanEval Pro and MBPP Pro: Evaluating Large Language Models on Self-invoking Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.21199","snapshot_observed_at":"2026-08-03T23:43:15.114501Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.114501Z"},"links":{"cited_paper":"/paper/2412.21199","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:71f7357e3635a871e5a437567d6091c8b5969814a1e8ffd3742bec1f2a56f6cd","observation_id":"3b213d7f-3344-4e7e-9e2a-b143a710c997","resolution":{"observed_at":"2026-08-03T23:43:15.114501Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-04T23:13:25.514661Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-03T23:43:15.170191Z","title":"Hendrycks, S","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.170191Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:917cdc8e21a564fc741b0e5b24761193f0be51747860f82c59f58fa772561d8f","observation_id":"9f913bb4-fb57-465d-95e6-93f62a46e99f","resolution":{"observed_at":"2026-08-03T23:43:15.170191Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:15.246548Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.246548Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:1ff7a165d56bb014c90f67045326419a1804fa6b76d197e702e34165df696325","observation_id":"ab594daf-a75d-4197-8369-128f0ea6b1d5","resolution":{"observed_at":"2026-08-03T23:43:15.246548Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-03T23:43:13.915869Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks"},"reference_resolution":{"displayed":11,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":11,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":11},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-04T06:34:03.388597+00:00","source":"crossref"},{"observed_at":"2026-08-04T06:33:57.428241+00:00","source":"retraction_watch"}],"thesis":"As of 5 August 2026, this Paper Citation Record lists 11 of 11 outbound references and 1 inbound Pith citation observation for arXiv:2511.04355."}