{"as_of":"2026-08-13T06:49:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:073d561b175b23f903e38c23a07a3a3b4c97c6575befb3523682dd4c74eb6057","coverage":[{"denominator":45,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":45,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T16:38:42.962606Z","state":"measured"},{"denominator":46,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":46,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T20:20:21.052795Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-11T00:16:15.729268Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"cited_work":{"arxiv_id":"2411.13323","doi":"10.48550/arxiv.2411.13323","metadata_source":"pith","pith_arxiv_id":"2411.13323","snapshot_observed_at":"2026-08-11T00:16:15.729268Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","venue":"cs.SE","work_id":"58940c65-7961-40bd-ad15-3eb2fd8b6629","year":2024},"citing_paper":{"arxiv_id":"2501.08840","last_updated":"2025-01-15T14:50:46Z","snapshot_observed_at":"2026-08-13T04:57:13.197089Z","submitted_at":"2025-01-15T14:50:46Z","title":"CveBinarySheet: A Comprehensive Pre-built Binaries Database for IoT Vulnerability Analysis","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-10T20:20:21.052795Z"},"links":{"cited_paper":"/paper/2411.13323","citing_paper":"/paper/2501.08840"},"observation_digest":"sha256:f154dfe030254d272cebf7aa08efd4087a11e5102ab0f548847c0137d996bf19","observation_id":"904278f3-d300-46fa-94f9-2b1ea3fc9164","resolution":{"observed_at":"2026-08-10T20:20:21.116359Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.13323/citation-record","integrity":"/paper/2411.13323/integrity","json":"/paper/2411.13323/citation-record.json","paper":"/paper/2411.13323"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.172401Z","title":"A survey on software fault localization,","venue":null,"work_id":"12302fcf-9ad4-4efa-8806-4aa9ced7e424","year":2016},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.252494Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:724b6f745a8f68b06812d02d9b8994d7d523329eb0dc1a3ff5181c3f46d32656","observation_id":"19cccd37-eae3-4fe3-9883-69a6b40c60dc","resolution":{"observed_at":"2026-08-12T16:38:44.177178Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.159488Z","title":"Automated Program Repair,","venue":null,"work_id":"85a01ede-f6d6-4fb7-8f2f-fa52c20cbd17","year":2019},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.256974Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:7fa14b92e6d826ddb24fda2700de52c7f83e65c8586475529a8819a5e8628c8a","observation_id":"619004af-2b05-49c6-aad8-7356b5ef0525","resolution":{"observed_at":"2026-08-12T16:38:44.163595Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.147236Z","title":"Defects4j: a database of existing faults to enable controlled testing studies for java programs,","venue":null,"work_id":"1341c7f9-68af-44bc-9420-034f5dc0f3f2","year":2014},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.261686Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:eb26b01aeea2dc14171dfe48c301fcde9a62db655c9e255ce0a0ed6f03af5215","observation_id":"9546f144-40e9-40a9-be7a-788c08e8949b","resolution":{"observed_at":"2026-08-12T16:38:44.151216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.134494Z","title":"Bugsinpy: a database of existing bugs in python programs to enable controlled testing and debugging studies,","venue":null,"work_id":"369f5c1a-fb9a-4e60-a22f-ee5fac78b2c2","year":2020},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.265987Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:3b9c4bd4df4c2e64da8fa2d291dcecca366ff94966bf00f5e65054c9011376ec","observation_id":"a52e983f-e0fe-4a17-b62e-0a3a1e26410c","resolution":{"observed_at":"2026-08-12T16:38:44.139126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.063979Z","title":"SWE-bench: Can language models resolve real-world github issues?","venue":null,"work_id":"5342ead1-427c-4332-9c58-057faacf3196","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.270348Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c63d6623ed71e0a89b5afaeb681c4d17edc15491d9ddfdd89bb305268b848456","observation_id":"d0bfa93d-fe52-4f03-8c37-1e0cf92ad63f","resolution":{"observed_at":"2026-08-12T16:38:44.124917Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.16898","last_updated":"2024-03-28T05:00:47Z","snapshot_observed_at":"2026-08-13T00:45:32.721908Z","submitted_at":"2024-03-25T16:10:25Z","title":"Concerned with Data Contamination? Assessing Countermeasures in Code Language Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.16898","snapshot_observed_at":"2026-08-12T16:38:42.274919Z","title":"Concerned with data contamination? assessing countermeasures in code language model,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.274919Z"},"links":{"cited_paper":"/paper/2403.16898","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:e5450eaf0a99b9a7ee16182bb926767199ae7b95008e0d96af46c7ed4d1dcd52","observation_id":"05131f45-ffc5-4890-a61f-3b941cf05c6b","resolution":{"observed_at":"2026-08-12T16:38:42.274919Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2207.07048","last_updated":"2022-07-14T16:44:59Z","snapshot_observed_at":"2026-08-13T04:45:59.816520Z","submitted_at":"2022-07-14T16:44:59Z","title":"Leakage and the Reproducibility Crisis in ML-based Science","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.07048","snapshot_observed_at":"2026-08-12T16:38:42.280170Z","title":"Leakage and the reproducibility crisis in ml-based science,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.280170Z"},"links":{"cited_paper":"/paper/2207.07048","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:5f8899fdb0d6afa5f30cf5140e088e354f4dc5ba1d76f3dca57f8e8dde864de2","observation_id":"e1c4d370-6e1e-4cf5-bcab-de488e646eed","resolution":{"observed_at":"2026-08-12T16:38:42.280170Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18824","last_updated":"2024-04-29T16:05:36Z","snapshot_observed_at":"2026-08-13T04:56:39.486630Z","submitted_at":"2024-04-29T16:05:36Z","title":"Benchmarking Benchmark Leakage in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18824","snapshot_observed_at":"2026-08-12T16:38:42.284289Z","title":"Benchmarking benchmark leakage in large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.284289Z"},"links":{"cited_paper":"/paper/2404.18824","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c6fa70d34059b29d93e9e60e6ef609e5314901b9c4ef4dd9fb928ff308847326","observation_id":"91903aa1-6de5-4571-be83-ce728ec2225b","resolution":{"observed_at":"2026-08-12T16:38:42.284289Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-08-13T00:56:54.856433Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-12T16:38:42.288484Z","title":"Estimating contamination via perplexity: Quantifying memo- risation in language model evaluation,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.288484Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:8d39766517834a2154988a5c443eba2ea2f7a08cb05799bd869349f678d577c0","observation_id":"4f9e806b-50c5-4845-a892-601a9d8ad91c","resolution":{"observed_at":"2026-08-12T16:38:42.288484Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.296648Z","title":"Bugsc++: A highly usable real world defect benchmark for c/c++,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.296648Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:3c5310263226997ed37ebb8f7fee8ea0f52d744f0f15dfeef27ee455f02915cf","observation_id":"9cdb5a41-7298-4f92-9321-46d76c4fce2f","resolution":{"observed_at":"2026-08-12T16:38:42.296648Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.884096Z","title":"Gitbug-java: A reproducible benchmark of recent java bugs,","venue":null,"work_id":"cf1aae2f-fe5c-4598-a38c-a6a0a06d3225","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.300069Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a69a476e04eed1aab14c8adecdf54e89f8c00fca0b9c2ce7315265d5a1f316a2","observation_id":"d09c4910-7504-420a-9664-9fc61b9e9c9c","resolution":{"observed_at":"2026-08-12T16:38:43.959940Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01489","last_updated":"2024-10-29T17:29:27Z","snapshot_observed_at":"2026-08-10T19:28:41.964638Z","submitted_at":"2024-07-01T17:24:45Z","title":"Agentless: Demystifying LLM-based Software Engineering Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01489","snapshot_observed_at":"2026-08-12T16:38:42.304045Z","title":"Agentless: Demystifying llm-based software engineering agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.304045Z"},"links":{"cited_paper":"/paper/2407.01489","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:b4beaded90afb259aa0b5bbf5b2a9d5f09fab9f0bcf2e5173904b5de3f94ae05","observation_id":"7fb3822f-891b-4631-97b0-6ce0bed8fcee","resolution":{"observed_at":"2026-08-12T16:38:42.304045Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.16741","last_updated":"2025-04-18T18:14:31Z","snapshot_observed_at":"2026-08-02T14:58:44.167588Z","submitted_at":"2024-07-23T17:50:43Z","title":"OpenHands: An Open Platform for AI Software Developers as Generalist Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.16741","snapshot_observed_at":"2026-08-12T16:38:42.308840Z","title":"Openhands: An open platform for ai software developers as generalist agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.308840Z"},"links":{"cited_paper":"/paper/2407.16741","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:ed947e0ad14c1d315047ed7959fc6ac58cb28402d252ffe760f6df067434a24a","observation_id":"9db61581-7331-4858-9b14-7e0d47a8eb4c","resolution":{"observed_at":"2026-08-12T16:38:42.308840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.06992","last_updated":"2024-10-10T13:13:09Z","snapshot_observed_at":"2026-08-12T22:27:08.380398Z","submitted_at":"2024-10-09T15:38:53Z","title":"SWE-Bench+: Enhanced Coding Benchmark for LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.06992","snapshot_observed_at":"2026-08-12T16:38:42.349704Z","title":"Swe-bench+: Enhanced coding benchmark for llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.349704Z"},"links":{"cited_paper":"/paper/2410.06992","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:7e843dc75fd23b7c99d40580270dc0b5528cdb656333eea4c2a20892126bc51b","observation_id":"734c1bc4-4623-40cb-aa05-49b7d4d376e7","resolution":{"observed_at":"2026-08-12T16:38:42.349704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.872184Z","title":"On the resemblance and containment of documents,","venue":null,"work_id":"a11073f1-cb94-44d1-a650-0ccc895eceb2","year":1997},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.405481Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:0d921cd31bfa25709e7f6047bde942961a1f610a01aa309d884a4fcf227aa87d","observation_id":"c6e344e4-9bad-4ae1-9a67-b4b9258ae866","resolution":{"observed_at":"2026-08-12T16:38:43.877092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.859668Z","title":"Similarity search in high dimensions via hashing,","venue":null,"work_id":"7ea8af59-ff7a-4253-9056-3d33bef456ab","year":1999},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.506527Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:49f7b49e981ca15968bf9df074eeb2ebfe1d5af05d419c24f106e28b28a67572","observation_id":"6242c7ba-190b-4661-ac2c-3fde315d8ec7","resolution":{"observed_at":"2026-08-12T16:38:43.863930Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.847687Z","title":"Codegen: An open large language model for code with multi-turn program synthesis,","venue":null,"work_id":"7894a460-4271-4d58-a989-f8b0a026af3f","year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.510429Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:7771b959314c4f398452780a12f37f9b3baaa65aa18471144a57b61f6f40ad1c","observation_id":"5233ba2b-6590-4991-bb27-c1ab79d191d5","resolution":{"observed_at":"2026-08-12T16:38:43.852055Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.833545Z","title":"Code llama: Open foundation models for code,","venue":null,"work_id":"a2307b40-eb96-42de-8bac-09c04736d517","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.514409Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:796c6472f920b5bb27c16e8db70a325ac3d82b55dfe115cdef133c8d6073f868","observation_id":"fcf00b6b-9882-4094-a865-574eb4dff3d6","resolution":{"observed_at":"2026-08-12T16:38:43.837616Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.522310Z","title":"Llama: Open and efficient foundation language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.522310Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:5b5b73a1d8c54c73344b60be3cca57e82fb3812b846a0e274c5954a1725ca7d6","observation_id":"5722db26-a0a7-4e0f-b402-92c10b750856","resolution":{"observed_at":"2026-08-12T16:38:42.522310Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.12950","last_updated":"2024-01-31T19:47:26Z","snapshot_observed_at":"2026-08-13T04:25:38.282910Z","submitted_at":"2023-08-24T17:39:13Z","title":"Code Llama: Open Foundation Models for Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.12950","snapshot_observed_at":"2026-08-12T16:38:42.518109Z","title":"Available: https://doi.org/10.48550/arXiv.2308.12950","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.518109Z"},"links":{"cited_paper":"/paper/2308.12950","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:8827b29c5c38b3046d19c1e9ae7c27984077e23a15684dcbb8d8b07120eeaa42","observation_id":"6b7abd74-868e-4562-a69b-ed465abbc3a7","resolution":{"observed_at":"2026-08-12T16:38:42.518109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-12T16:38:42.530024Z","title":"Gemma 2: Improving open language models at a practical size,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.530024Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:e76bd5c23986793e65734ceed313fa1cc372785516ada9d62e12ff127fceeefd","observation_id":"5648e50f-7837-4773-8691-f79a815bf547","resolution":{"observed_at":"2026-08-12T16:38:42.530024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.694587Z","title":"Starcoder 2 and the stack v2: The next generation,","venue":null,"work_id":"c5aec53f-3b40-4c09-b336-98141aed9321","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.525883Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:0f4b57ca950488d943659e9e616d20d657b5eb1a8437586a2a546cd02cf41c6a","observation_id":"6a8e0c35-14a6-47d8-9314-ecbfd02df6bf","resolution":{"observed_at":"2026-08-12T16:38:43.776154Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06825","last_updated":"2023-10-10T17:54:58Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-10-10T17:54:58Z","title":"Mistral 7B","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06825","snapshot_observed_at":"2026-08-12T16:38:42.758008Z","title":"Mistral 7b,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.758008Z"},"links":{"cited_paper":"/paper/2310.06825","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:083b18cec3d133cb690a53d97b7581814aacc8cf83038d5c34138ec7908e9061","observation_id":"c97a4665-f0e4-47f3-a172-c052093eb1fa","resolution":{"observed_at":"2026-08-12T16:38:42.758008Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.682270Z","title":"Codegemma: Open code models based on gemma,","venue":null,"work_id":"ccbefe3b-c11e-4e78-9a59-40aefa98e931","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.586465Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:1bfc749acca849b15389c457446f2e370ed58941e61ae36b256c53ed59ca02ed","observation_id":"29f57848-3c77-446d-ac1a-2e9bf18ee480","resolution":{"observed_at":"2026-08-12T16:38:43.686661Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.656119Z","title":"Repairllama: Efficient representations and fine-tuned adapters for program repair,","venue":null,"work_id":"ce05012c-1402-450d-a733-11c40dee1c34","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.795695Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:934c8a5bde984a3588e7f364b284d250a85b53a69da5df9751386e448c0ce0bd","observation_id":"5f6b8255-4788-4973-bdc5-e68bb0434e98","resolution":{"observed_at":"2026-08-12T16:38:43.660234Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.642785Z","title":"Large language model for vulnerability detection: Emerging results and future directions,","venue":null,"work_id":"784215a1-8530-47a3-9ab5-33b8f35c2a10","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.804097Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:31867bad1e6bb3609dffcf949a8b3277f3fb6ce0df5fa2bc3168f78de47d2c0a","observation_id":"1a7cc93e-7ad8-43a2-8bcb-3823f9b331c6","resolution":{"observed_at":"2026-08-12T16:38:43.647047Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.668944Z","title":"Large language models for test-free fault localization,","venue":null,"work_id":"55b9921b-2e14-4ffa-b548-686d8ee71c27","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.791692Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:1ef56e9ac74e8262720bd77575c8834058163b461246418800391adb2757d470","observation_id":"10514c65-8456-4621-9750-548c46acfb89","resolution":{"observed_at":"2026-08-12T16:38:43.673850Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.464715Z","title":"A study of the uniqueness of source code,","venue":null,"work_id":"d80f8d93-ed35-48ae-a0bd-744e01f98699","year":2010},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.811382Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:7ea18b84e3e5bfa9ae877434ea380096c1551b073fa6ff22918d973392521386","observation_id":"51d34f47-928c-4ef0-b6c9-2fea25c1c71e","resolution":{"observed_at":"2026-08-12T16:38:43.468959Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.15698","last_updated":"2025-09-03T01:34:48Z","snapshot_observed_at":"2026-08-13T04:54:15.222217Z","submitted_at":"2023-12-25T11:39:46Z","title":"RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.15698","snapshot_observed_at":"2026-08-12T16:38:42.799148Z","title":"Available: https://doi.org/10.48550/arXiv.2312.15698","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.799148Z"},"links":{"cited_paper":"/paper/2312.15698","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:6155281cffe984c06ab8222afbc4c07a7bbcb425da55eca91a7989c66d415ab7","observation_id":"59e1695b-2a69-4bf8-ab28-b7dafbd097c9","resolution":{"observed_at":"2026-08-12T16:38:42.799148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.450065Z","title":"Memorization without overfitting: Analyzing the training dynamics of large language models,","venue":null,"work_id":"0d765e7a-192a-4ac1-a141-3d9ab3e22db2","year":2022},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.819025Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:f5c2decdc7de684cadb2b6bd00961b408928151dec5b1f97a5393c7f661e657b","observation_id":"49ac9234-caa2-4cd3-aa83-76a8f17f36c4","resolution":{"observed_at":"2026-08-12T16:38:43.455722Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.492256Z","title":"The stack: 3 TB of permissively licensed source code,","venue":null,"work_id":"3aa96bc3-6c04-4443-afe7-2b2e2d0a2917","year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.807962Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a9c411a2371e8c56e189aa1160c5f24afabda1e39b9f2f10ebae55c6e71c3435","observation_id":"41c367a4-fe6c-4c5f-af46-ae0e0176eb1d","resolution":{"observed_at":"2026-08-12T16:38:43.575402Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.435696Z","title":"Measuring massive multitask language understanding,","venue":null,"work_id":"4c3303b7-a062-4f17-a2d5-2bd24ce4ed38","year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.826381Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:0780953385e40fe72606d0b69161ea36bdbe0d4af36c510153275025533ee47e","observation_id":"5b037e62-a0b0-4319-9405-f770df87ecad","resolution":{"observed_at":"2026-08-12T16:38:43.440137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08747","last_updated":"2025-01-05T04:09:00Z","snapshot_observed_at":"2026-08-10T01:52:30.559515Z","submitted_at":"2023-08-17T02:53:23Z","title":"An Empirical Study of Catastrophic Forgetting in Large Language Models During Continual Fine-tuning","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08747","snapshot_observed_at":"2026-08-12T16:38:42.814911Z","title":"An empirical study of catastrophic forgetting in large language models during continual fine-tuning,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.814911Z"},"links":{"cited_paper":"/paper/2308.08747","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:ac60395e46d77fdf31d02adf17cb6601ce76bc021addc3ee0a5d5b4e26c23ba6","observation_id":"aaaadc1c-7b6a-4bde-b9b6-f7a4fdaaff5e","resolution":{"observed_at":"2026-08-12T16:38:42.814911Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-12T16:38:42.838268Z","title":"Program synthesis with large language models,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.838268Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:4107d810e98bcffe11b073487b9597158e7d7c016198ff65a163d1deb35c2bb1","observation_id":"ccc78991-03b9-4c7d-9762-de97282f6fd7","resolution":{"observed_at":"2026-08-12T16:38:42.838268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.822872Z","title":"Squad: 100, 000+ questions for machine comprehension of text,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.822872Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:172ddf1883fcd60553b12930b5ebf3df6d1db1f7b0650cfc9637e68256b96e60","observation_id":"a1acba78-afc4-4fa8-8b08-09bdf13a6c33","resolution":{"observed_at":"2026-08-12T16:38:42.822872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.00385","last_updated":"2024-12-09T20:45:33Z","snapshot_observed_at":"2026-08-07T04:08:23.682670Z","submitted_at":"2023-04-01T20:57:33Z","title":"Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.00385","snapshot_observed_at":"2026-08-12T16:38:42.845327Z","title":"Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.845327Z"},"links":{"cited_paper":"/paper/2304.00385","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:bfa6b68add423f2488ba66a1717f8bd71ee41ea65fb34a751798f0589c2d89fc","observation_id":"7d5cf801-2987-42b0-9aff-464a161cbd97","resolution":{"observed_at":"2026-08-12T16:38:42.845327Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.830570Z","title":"Evaluating large language models trained on code,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.830570Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a6ea222e2cb7b62c57615ce39bf68ecadc54ce54abe71254f854ac086313a6d5","observation_id":"a3a57390-4812-4472-819f-e5750cd47e0a","resolution":{"observed_at":"2026-08-12T16:38:42.830570Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-10T03:07:35.408836Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-12T16:38:42.893040Z","title":"Livecodebench: Holistic and contamination free evalua- tion of large language models for code,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.893040Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:b28bf1827363ba8b8e464c9c2099ec375032bd170ce18db673c3a324f4662641","observation_id":"b293650d-04a7-415b-9989-f12de4f16fe7","resolution":{"observed_at":"2026-08-12T16:38:42.893040Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.303483Z","title":"Why does your data leak? uncovering the data leakage in cloud from mobile apps,","venue":null,"work_id":"4c908478-e445-43b2-9131-876a65c4b48c","year":2019},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.962606Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:9506ccb76abef7a90b46a74461cbc4c9d22842c528c4f6ab4c28e95a4936ec71","observation_id":"61797f08-cfb1-4f59-b0fd-409ea0d30555","resolution":{"observed_at":"2026-08-12T16:38:43.353535Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.414093Z","title":"Measuring coding challenge competence with APPS,","venue":null,"work_id":"ad9e8876-2fea-4113-af05-a5e05b0d1bbb","year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.841580Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:6391e7fd5f907b0783c4fb7e392fcb368c320cce0b138bf1ead82c58b650824c","observation_id":"47de63b2-7354-4d2d-944e-93e212e32bf4","resolution":{"observed_at":"2026-08-12T16:38:43.418581Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.400852Z","title":"Detecting pretraining data from large language models,","venue":null,"work_id":"23cefabc-4f39-4e6a-a3e2-80345c120da4","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.849344Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:475f2eef76b820ff4b116d6a3944ea9e2248b83fd504a8bfce5931f69f3981b9","observation_id":"aa29b1cc-1416-4b5b-b1ef-8677991695f3","resolution":{"observed_at":"2026-08-12T16:38:43.405143Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-10T03:07:35.408836Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-12T16:38:42.958319Z","title":"Available: https://doi.org/10.48550/arXiv.2403.07974","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.958319Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:ae659c7f2412bc20670afeae3cfed3225899a470d859dd5d61adc8fbf968d4b8","observation_id":"3b338049-eaca-4c1a-bed3-2f8252848170","resolution":{"observed_at":"2026-08-12T16:38:42.958319Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-12T16:38:42.833805Z","title":"Available: https://arxiv.org/abs/2107.03374","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.833805Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:da0944c32387e79fbd5f2d6e8f119331a5d8fbeae121a3b52a686cdb6869b59f","observation_id":"371f832b-16fd-45fa-8db1-91e2625b18a2","resolution":{"observed_at":"2026-08-12T16:38:42.833805Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-08-13T00:56:54.856433Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-12T16:38:42.292215Z","title":"Available: https://doi.org/10.48550/arXiv.2309.10677","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.292215Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:26601ebb1a03227ddbe8254a081943576198f8d54e3799657ba29a25f126c883","observation_id":"860f927a-9d1b-4c35-b4fc-9d5cea66c2cc","resolution":{"observed_at":"2026-08-12T16:38:42.292215Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11409","last_updated":"2024-06-19T02:37:50Z","snapshot_observed_at":"2026-08-12T23:41:14.320216Z","submitted_at":"2024-06-17T10:54:35Z","title":"CodeGemma: Open Code Models Based on Gemma","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11409","snapshot_observed_at":"2026-08-12T16:38:42.656619Z","title":"Available: https://doi.org/10.48550/arXiv.2406.11409","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.656619Z"},"links":{"cited_paper":"/paper/2406.11409","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:1f147b9960c931e4378a5ba2fdccb1c1afbbf810ebbaf49e9fbbd036ba0c70e8","observation_id":"df50b642-a106-4b59-b337-7ca5e5baebda","resolution":{"observed_at":"2026-08-12T16:38:42.656619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","latest_version":3,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-13T04:57:00.540234Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?"},"reference_resolution":{"displayed":45,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":23,"verified_exact":0,"verified_fuzzy":22},"total_outbound_references":45},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 45 of 45 outbound references and 1 inbound Pith citation observation for arXiv:2411.13323."}