{"as_of":"2026-08-08T08:28:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:8094aea162bcdbb453c9e44cdc5c62b3f255858610d4b9a93c52532ca84cf7d0","coverage":[{"denominator":22,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":22,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-14T19:54:08.600196Z","state":"measured"},{"denominator":22,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":22,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2604.03922/citation-record","integrity":"/paper/2604.03922/integrity","json":"/paper/2604.03922/citation-record.json","paper":"/paper/2604.03922"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T04:13:14.044604Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Evaluating large language models trained on code.Arxiv Preprint Arxiv:2107.03374,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:d630645bfd72cc7a663fbbcdbc8bb69a7f06add7bf0afddc01f09e6c147f4499","observation_id":"5ccbcc63-fc20-47a4-a144-4845bf20a1f0","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.14196","last_updated":"2024-01-26T09:23:11Z","snapshot_observed_at":"2026-08-06T22:40:28.707813Z","submitted_at":"2024-01-25T14:17:53Z","title":"DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.14196","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"DeepSeek-Coder: When the large language model meets programming–the rise of code intelligence.Arxiv Preprint Arxiv:2401.14196,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2401.14196","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:1718e004a699bd8f25a46b384e720e98b4e407b4388ae07e2a56c859d3699072","observation_id":"af18d80c-a96e-49a2-8b39-2db879824143","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21787","last_updated":"2024-12-30T19:03:24Z","snapshot_observed_at":"2026-07-06T18:55:11.576666Z","submitted_at":"2024-07-31T17:57:25Z","title":"Large Language Monkeys: Scaling Inference Compute with Repeated Sampling","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21787","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Large language monkeys: Scaling inference compute with repeated sampling.Arxiv Preprint Arxiv:2407.21787,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2407.21787","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:3c5c4834dcb42f2acf44d687ee9f06892730de275e9470f1fe99b3721512105a","observation_id":"9b67500c-1941-4499-9f6e-e181e8525835","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:24b2db309f858d1390349b3a8f65d4a65780f9dc5e5b9bfdf93ac4af115adbc8","observation_id":"f18b6b2d-1096-4701-89f5-fddbb4379aa4","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:c2b345b1d1500dbc529a8ae34d60c32852b524e0535c1d4c70b0c9653d9157a1","observation_id":"d4dfc75f-3674-4e4f-97d1-157fd84648a9","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Program synthesis with large language models.Arxiv Preprint Arxiv:2108.07732,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:e2a4ea7b056f4dc289481c31565cbe6abcd23419c1be9c7371d200f12f90832b","observation_id":"a5f174c3-cf4f-41dc-837e-842d525f398f","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"GPT-4 technical report.Arxiv Preprint Arxiv:2303.08774,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:1bc971a4febe094e5abf4ff07361a1e759ae65be4018c09597b5a4b102fffe70","observation_id":"131fc642-8a86-4e61-be5c-8f23f9cd2b4d","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.12950","last_updated":"2024-01-31T19:47:26Z","snapshot_observed_at":"2026-07-06T16:10:07.931347Z","submitted_at":"2023-08-24T17:39:13Z","title":"Code Llama: Open Foundation Models for Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.12950","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Code llama: Open foundation models for code.Arxiv Preprint Arxiv:2308.12950,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2308.12950","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:404ba6590356c90494726ebe9e24fbe92b10b493cfd87413fd3c8d21c54bebe3","observation_id":"56d0ef67-af6d-45bd-b001-d1caa998851a","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.19173","last_updated":"2024-02-29T13:53:35Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-02-29T13:53:35Z","title":"StarCoder 2 and The Stack v2: The Next Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.19173","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Starcoder 2 and the stack v2: The next generation.Arxiv Preprint Arxiv:2402.19173,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2402.19173","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:b31fd282b19bcb7ca73a2060d982ef92d0e2917b65a851d1a86e4d0e501b0a89","observation_id":"211c770c-d1c8-48c1-8e39-a503354527b6","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.12186","last_updated":"2024-11-12T13:24:25Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-09-18T17:57:57Z","title":"Qwen2.5-Coder Technical Report","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.12186","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2409.12186","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:ca5a28931fe1ba859860e23fb302dc52b3aba9a20716285ad4ebf0465c7adf9c","observation_id":"901497dc-8554-4741-a659-d5c242093761","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Sampling-based approximations to minimum Bayes risk decoding for neural machine translation","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:976431e574c8ea47d5ff427d3697f274280f29e3834946946638137c87ae181c","observation_id":"6c9c9d34-478b-4258-bc47-1969b596ba22","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"ALGO: Synthesizing algorithmic programs with generated oracle verifiers","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:b6e89f988d8c154c6fc2d0535bb120419a8ccd836c8899074cc871c8f5f80bd4","observation_id":"7abc866b-1047-4832-8a6e-0428af205661","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Consistency meets verification: Enhancing test generation quality in large language models without ground-truth solutions.Arxiv Preprint Arxiv:2602.10522,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:60d486742f3149030972e84495f758306689f737967f813ddb82c6d53363f026","observation_id":"dc0e439a-d2d8-4e2f-9234-e633ec6b69e3","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13745","last_updated":"2024-10-16T15:07:41Z","snapshot_observed_at":"2026-07-06T19:05:36.210400Z","submitted_at":"2024-08-25T07:10:36Z","title":"DOCE: Finding the Sweet Spot for Execution-Based Code Generation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13745","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"DOCE: Finding the sweet spot for execution- based code generation.Arxiv Preprint Arxiv:2408.13745,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2408.13745","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:20a3c0de4d4a11f65d3a6420dc983ce2e2dd9f5471cd4d513f4866d27229db19","observation_id":"cae1fb30-1f0d-401f-93fc-39df798c1181","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2208.02126","last_updated":"2022-08-17T15:12:44Z","snapshot_observed_at":"2026-08-05T21:10:41.235201Z","submitted_at":"2022-08-03T15:04:48Z","title":"Noise tolerance of learning to rank under class-conditional label noise","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.02126","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Noise tolerance of learning to rank under class-conditional label noise.Arxiv Preprint Arxiv:2208.02126,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2208.02126","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:f01dc28111751d1c6bd37ca93ec4ce2bfd0b72bfcfc00a0f2ad239c104798753","observation_id":"6753e716-1c19-4582-8b37-a1480cfdc85b","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Training verifiers to solve math word problems, 2021.Arxiv Preprint Arxiv:2110.14168,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:446dba493d4876842a2c3f2e4fa4b59d60ccec84d5487fa67a4ce06fcb7758a0","observation_id":"b340f575-376c-445c-aa55-66cc645d5afe","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.14655","last_updated":"2025-04-20T15:28:16Z","snapshot_observed_at":"2026-08-07T16:00:46.114758Z","submitted_at":"2025-04-20T15:28:16Z","title":"LeetCodeDataset: A Temporal Dataset for Robust Evaluation and Efficient Training of Code LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.14655","snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Leet- CodeDataset: A temporal dataset for robust evaluation and efficient training of code LLMs.arXiv preprint arXiv:2504.14655,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"cited_paper":"/paper/2504.14655","citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:9f7e25ce0e5dcb319a52217a0876fbb17b376ec4eabcd71f58d8efc6ae4ae7ec","observation_id":"7d1c0462-b7d0-4a3f-9062-d1db139bbf3d","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"It follows that the true AUC A(w) is the same for every pair (c+ ∈C +, c − ∈C −) and that two codes from the same class have identically distributed scores","venue":null,"work_id":null,"year":1963},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:2294ead9b026ba413040988bcc94c5760f08fd905f59b803f72767e86ed535db","observation_id":"36b95787-e9f9-49c6-9213-8fa8cc1121c8","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"Under uniform initial weights, the leave-one-out scores reduce to unweighted row sums with one column removed; since the AUC is scale-invariant, normalization is unnecessary","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:3cfb882ad925e9370a8413ab4177ebcb928bd62e62e4a946a05b9e14ba3b028b","observation_id":"c73d5b24-1062-43ec-8f6c-fab6d11f1050","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:327bbd7c6299e85c997514e86891903764ac8838a3e887e1fe59b362717a9129","observation_id":"24c2a085-c8df-4e5e-81eb-6fcd2b2111eb","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"correct(α j =1)incorrect(α j <1) perf","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:05e692408cba84aad86990d4d4273404a4d236b0aba12d92dcba182ad51063aa","observation_id":"a75c7d0c-7173-4710-bfd8-62efb9807525","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T19:54:08.600196Z","title":"assert {entry point}(...) ==","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-07-14T19:54:08.600196Z"},"links":{"citing_paper":"/paper/2604.03922"},"observation_digest":"sha256:d7c9b7595f6f2e1f280d48bd2da3c57081f46fe8b69080a406a78873a403b394","observation_id":"75890a00-ed46-4d6a-868e-fc8c972131e1","resolution":{"observed_at":"2026-07-14T19:54:08.600196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2604.03922","last_updated":"2026-07-12T05:30:15Z","latest_version":2,"primary_category":"cs.LG","snapshot_observed_at":"2026-07-16T23:18:41.605526Z","submitted_at":"2026-04-05T01:15:57Z","title":"ACES: Who Tests the Tests? Leave-One-Out AUC Consistency for Code Generation"},"reference_resolution":{"displayed":22,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":21,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":22},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 22 of 22 outbound references and 0 inbound Pith citation observations for arXiv:2604.03922."}