{"as_of":"2026-08-17T19:48:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:6fde26c2aa0c0ca32793dc5d32755fc8f5cec587d5dc30f28a4aff83009e8edf","coverage":[{"denominator":56,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":56,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T22:00:10.470063Z","state":"measured"},{"denominator":57,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":57,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-20T07:19:50.354875Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-20T07:23:07.041987Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"cited_work":{"arxiv_id":"2505.08389","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.08389","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv preprint arXiv:2505.08389 , year =","venue":null,"work_id":"4d14f494-0836-453e-8d9d-007250c19ddd","year":null},"citing_paper":{"arxiv_id":"2605.19999","last_updated":"2026-05-19T15:33:16Z","snapshot_observed_at":"2026-08-15T01:06:56.797913Z","submitted_at":"2026-05-19T15:33:16Z","title":"LLM Benchmark Datasets Should Be Contamination-Resistant","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-05-20T07:19:50.354875Z"},"links":{"cited_paper":"/paper/2505.08389","citing_paper":"/paper/2605.19999"},"observation_digest":"sha256:85c893588cb89f1cce8bfc9cd27cfdb7dafa20eac34d42e9beb41ca21626ae04","observation_id":"eeba85aa-00bb-43c1-af8b-210a0fb75436","resolution":{"observed_at":"2026-05-20T07:23:07.043308Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2505.08389/citation-record","integrity":"/paper/2505.08389/integrity","json":"/paper/2505.08389/citation-record.json","paper":"/paper/2505.08389"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.035486Z","title":"What learning algorithm is in-context learning? investigations with linear models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.035486Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:24e96b4b89e09f98fd7f3a43b40608beeec299c4db696ee8f6a1e87809eb39e6","observation_id":"69940e1e-cddd-4802-a9de-6a158853e6e8","resolution":{"observed_at":"2026-08-15T22:00:10.035486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.07279","last_updated":"2025-03-25T03:36:21Z","snapshot_observed_at":"2026-08-16T13:01:03.265718Z","submitted_at":"2024-11-11T18:59:45Z","title":"The Surprising Effectiveness of Test-Time Training for Few-Shot Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.07279","snapshot_observed_at":"2026-08-15T22:00:10.060656Z","title":"The surprising effectiveness of test-time training for abstract reasoning","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.060656Z"},"links":{"cited_paper":"/paper/2411.07279","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:da22074d6f46e6810180c35e2707d9aaf18466b30cc9422d4340395832206d35","observation_id":"fb7d25df-a325-4e73-a1b0-4198f9551556","resolution":{"observed_at":"2026-08-15T22:00:10.060656Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.360005Z","title":"Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source llms","venue":null,"work_id":"b804a167-89b8-4bac-b4fa-7341d27c200e","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.096677Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:203466d04d9f00b143064f20fbb1d750e3c6d9159bdda905b82ae1c20e7c2365","observation_id":"679693a7-2f00-4adf-bb6c-f512b27a3bbb","resolution":{"observed_at":"2026-08-15T22:00:11.363940Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.106629Z","title":"Do, Yan Xu, and Pascale Fung","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.106629Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:46f72fae58e6671ffdf34a0a95c7f10fd4d585cb51f280f8a54bb479e4a325ad","observation_id":"61240b48-40e3-4d10-b95e-e38734ebd56b","resolution":{"observed_at":"2026-08-15T22:00:10.106629Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.114432Z","title":"Managing extreme ai risks amid rapid progress","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.114432Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:6ea65763495b04fac0864f0f285ddb16eda7a7347726b59c15ea25ced11d7a0b","observation_id":"da1627ff-46a5-47f4-b407-c5ffef23943e","resolution":{"observed_at":"2026-08-15T22:00:10.114432Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.326411Z","title":"Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, et al","venue":null,"work_id":"1617db01-3913-405c-95ff-410c978e6253","year":2020},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.126612Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:54e15dac09055b7bacb6dcffbc35c4dab5a72be2935534b1ed6b6d6a4346fe1a","observation_id":"96c32ab6-d82b-4475-b5a9-ce0a2804a633","resolution":{"observed_at":"2026-08-15T22:00:11.332096Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.12712","last_updated":"2023-04-13T20:41:31Z","snapshot_observed_at":"2026-08-03T04:49:15.195814Z","submitted_at":"2023-03-22T16:51:28Z","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.12712","snapshot_observed_at":"2026-08-15T22:00:10.135591Z","title":"Lundberg, Harsha Nori, Hamid Palangi, Marco T \\' u lio Ribeiro, and Yi Zhang","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.135591Z"},"links":{"cited_paper":"/paper/2303.12712","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:0f014059192d6fb7da5103568882a87454624c167bea967198609f8cbaf1f864","observation_id":"a222112e-aace-4c1a-bd86-27d64ddfa00c","resolution":{"observed_at":"2026-08-15T22:00:10.135591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.00393","last_updated":"2024-06-24T08:28:18Z","snapshot_observed_at":"2026-08-16T14:13:49.506608Z","submitted_at":"2024-03-01T09:28:38Z","title":"TRUCE: Private Benchmarking to Prevent Contamination and Improve Comparative Evaluation of LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.00393","snapshot_observed_at":"2026-08-15T22:00:10.141863Z","title":"Private benchmarking to prevent contamination and improve comparative evaluation of llms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.141863Z"},"links":{"cited_paper":"/paper/2403.00393","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:bbdfa740acb4f2588abe2f991966f71e616ab46f01c034bb52f0e6dfac1185fd","observation_id":"9afb45fe-9725-4a00-953f-ba4af404d928","resolution":{"observed_at":"2026-08-15T22:00:10.141863Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.309730Z","title":"Palm: Scaling language modeling with pathways","venue":null,"work_id":"37bcb9ab-d6c9-4f2c-8f4d-708213a864a0","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.156182Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:402c7f43085d062b8c836f343b1fec8420df03b721ca0c09407498e572a29b20","observation_id":"87c02f98-6247-419a-8ac1-f330f1090157","resolution":{"observed_at":"2026-08-15T22:00:11.315065Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.11416","last_updated":"2022-12-06T21:39:48Z","snapshot_observed_at":"2026-08-17T10:20:11.128461Z","submitted_at":"2022-10-20T16:58:32Z","title":"Scaling Instruction-Finetuned Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.11416","snapshot_observed_at":"2026-08-15T22:00:10.164195Z","title":"Scaling instruction-finetuned language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.164195Z"},"links":{"cited_paper":"/paper/2210.11416","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:7cf1058089b2066daac75bbe7ac30ea7c843247e3d2cf7e610b77ef9b1305b7b","observation_id":"9b19019a-fbde-4bf2-9906-fbb9ef6bf8e6","resolution":{"observed_at":"2026-08-15T22:00:10.164195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-15T22:00:10.171454Z","title":"Training verifiers to solve math word problems","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.171454Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:2cda99eaf6337feea9507de4c3167dfb6e1e0e0acfb73b117503652c0440f55a","observation_id":"5a102519-49cd-4d08-8674-c0cfcf141338","resolution":{"observed_at":"2026-08-15T22:00:10.171454Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.294978Z","title":"Why can GPT learn in-context? language models secretly perform gradient descent as meta-optimizers","venue":null,"work_id":"cd4ce4e4-1c6d-4560-80cb-40d5d27e6246","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.179869Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:8e434367afca51434a30b1f2a9a46464125cfaadd8b5b03ad80a698d87395986","observation_id":"67c3b58c-0549-4070-9353-333d8b900a68","resolution":{"observed_at":"2026-08-15T22:00:11.300322Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.279368Z","title":"Generalization or memorization: Data contamination and trustworthy evaluation for large language models","venue":null,"work_id":"eec219e9-47fb-4a90-9f2d-906fa82b4cbe","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.185895Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:2e175dafaa7d4a1d10d8ee40ba5b70cbbbbeda467b6d4fb31972d1c1c4d39472","observation_id":"0db4f8e0-25b6-4f1f-ae6c-6bb1978294c1","resolution":{"observed_at":"2026-08-15T22:00:11.284886Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-15T22:00:10.190458Z","title":"The llama 3 herd of models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.190458Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:7a5d299072d4f913e80e027421d6702d9caaaaf88f3c6f6c0d2bc5ca22ffc7ca","observation_id":"9682585c-cd52-4507-93c8-ea1ec6a6b3c6","resolution":{"observed_at":"2026-08-15T22:00:10.190458Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.eacl-main.7","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.783469Z","title":"Cole, Fangyu Liu, and William W","venue":null,"work_id":"17090fcc-3923-4d46-91a4-fadd14b2ac65","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.196731Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:cf280d8e30b60958c69619b1c4d21a320801c8883f6cf6a00f6eaf69ca5e42b8","observation_id":"57bfe173-e544-4f93-879f-1cadb603ae5a","resolution":{"observed_at":"2026-08-15T22:00:10.787781Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.264618Z","title":"What can transformers learn in-context? A case study of simple function classes","venue":null,"work_id":"d5053b64-484d-4e29-8e58-a647b0a53b3b","year":2022},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.201762Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f8cd79b80d764c8ff994ab2fbed3fbcf13f9e708f1550625aa2024f071779fd4","observation_id":"06e56a22-f3b8-4d18-943a-df202aaec5ab","resolution":{"observed_at":"2026-08-15T22:00:11.269783Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.249889Z","title":"Human-like intuitive behavior and reasoning biases emerged in large language models but disappeared in chatgpt","venue":null,"work_id":"f714d522-0103-46d5-89e4-212cebb5f48e","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.207651Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:39df86738c39384a61d6ad522bc6816601d409bf262a82e5605cde4a24c58542","observation_id":"78704c4c-d05c-4b92-b5ac-329530667238","resolution":{"observed_at":"2026-08-15T22:00:11.254168Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.213556Z","title":"Instructed to Bias: Instruction-Tuned Language Models Exhibit Emergent Cognitive Bias","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.213556Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:7fef00f73372aae78c6831b9a5ec7e4eee1eb91f9f42d2ab3469770b8f20c300","observation_id":"9c551f1c-83c8-45a8-9a2d-e3c0714bbc48","resolution":{"observed_at":"2026-08-15T22:00:10.213556Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-16T07:05:57.323612Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-15T22:00:10.217836Z","title":"Livecodebench: Holistic and contamination free evaluation of large language models for code","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.217836Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f68ae02ffcb65de6fa59e5b6a6a5467dc2f6b927d7574d8f1346e23a23be555d","observation_id":"0e3b71f0-7d93-4473-815c-50eda4a52d82","resolution":{"observed_at":"2026-08-15T22:00:10.217836Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.234422Z","title":"Does data contamination make a difference? insights from intentionally contaminating pre-training data for language models","venue":null,"work_id":"6d61c217-c123-42e7-a0b9-f922f6ced6c2","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.224057Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:c68e009c25fc3a64c0a43155541dff1f06b28caf1d0a6ebbc8153b4a20c23331","observation_id":"ff868702-fe1a-423e-99fa-885499a739df","resolution":{"observed_at":"2026-08-15T22:00:11.240666Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.220653Z","title":"Large language models are zero-shot reasoners","venue":null,"work_id":"a2a9af89-4787-4887-89ff-a58aa72fd477","year":2022},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.256194Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:a1179e9e37d7ef01d8cb67e60a6c7a2838fb73b6f38837a7ad16f116965df22e","observation_id":"faa7495c-5684-4147-86ab-518b2fae3cc3","resolution":{"observed_at":"2026-08-15T22:00:11.224617Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.263224Z","title":"Task contamination: Language models may not be few-shot anymore","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.263224Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:537f2419731948c154ebc7c3fc2c392aa83c972ff4c4f320ad2bd48c48f3a881","observation_id":"120ad59c-f67d-498d-af83-09969a6ee94a","resolution":{"observed_at":"2026-08-15T22:00:10.263224Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.11553","last_updated":"2024-12-11T09:04:18Z","snapshot_observed_at":"2026-08-16T18:16:41.360493Z","submitted_at":"2024-04-17T16:53:16Z","title":"Language Ranker: A Metric for Quantifying LLM Performance Across High and Low-Resource Languages","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.11553","snapshot_observed_at":"2026-08-15T22:00:10.269455Z","title":"Quantifying multilingual performance of large language models across languages","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.269455Z"},"links":{"cited_paper":"/paper/2404.11553","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:3f837dc7ac7c81b2f7017b45a13e1d3c0861b7fb9d432b25c3c9035e421228d1","observation_id":"c769f84a-a61c-44a5-a3c1-355cca4e40c3","resolution":{"observed_at":"2026-08-15T22:00:10.269455Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.198802Z","title":null,"venue":null,"work_id":"88c1b358-195c-431c-924e-d2b21908b4d3","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.274302Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:5ad257e664bd583e71ef4745506ca971b81dc616951f6ae5865d47704e260bdc","observation_id":"8a2aba1a-0999-4c81-b000-1a3fe0e4a1df","resolution":{"observed_at":"2026-08-15T22:00:11.202371Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.183814Z","title":null,"venue":null,"work_id":"52099ebd-6fdb-46f1-8500-7da12c5db4da","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.280041Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:5bd6074623cadb8bb10bad2b7e98c36d3a4d4617d447a4fe63ccb361c7d0cb30","observation_id":"91be474c-debb-4978-af9e-e49d7067a69e","resolution":{"observed_at":"2026-08-15T22:00:11.188688Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.14275","last_updated":"2025-06-27T02:05:51Z","snapshot_observed_at":"2026-08-16T05:55:51.647767Z","submitted_at":"2025-01-24T06:39:38Z","title":"Leveraging Online Olympiad-Level Math Problems for LLMs Training and Contamination-Resistant Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.14275","snapshot_observed_at":"2026-08-15T22:00:10.289031Z","title":"Leveraging online olympiad-level math problems for llms training and contamination-resistant evaluation","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.289031Z"},"links":{"cited_paper":"/paper/2501.14275","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:86c9992dd5f86c33369a0f505447dcff475c0a49cd1bc08a6b73c8f13a772edf","observation_id":"f81a0d5a-71e7-493d-802d-61f26c9b7f61","resolution":{"observed_at":"2026-08-15T22:00:10.289031Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.293932Z","title":"Thomas McCoy, Shunyu Yao, Dan Friedman, Mathew D","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.293932Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:29c9ef8a0e9f9217e61d1c11bcb4a4b3239658b7a5fdd7fa7c48caf73dc6e66c","observation_id":"3a828076-cf65-4803-bece-354477b49413","resolution":{"observed_at":"2026-08-15T22:00:10.293932Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.01792","last_updated":"2024-10-04T03:57:33Z","snapshot_observed_at":"2026-08-16T13:13:15.346059Z","submitted_at":"2024-10-02T17:50:19Z","title":"When a language model is optimized for reasoning, does it still show embers of autoregression? An analysis of OpenAI o1","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.01792","snapshot_observed_at":"2026-08-15T22:00:10.298851Z","title":"Thomas McCoy, Shunyu Yao, Dan Friedman, Mathew D","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.298851Z"},"links":{"cited_paper":"/paper/2410.01792","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:4d3572b758afd74c04796fffd27afb3c580f487453f35f3f32b32d02a79864f8","observation_id":"3fd22f3c-3528-4a6a-8fc5-7c039a60179a","resolution":{"observed_at":"2026-08-15T22:00:10.298851Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.303348Z","title":"Sources of hallucination by large language models on inference tasks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.303348Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:8139aebdff1fcf4fe3946239bce5ac5d4567d279db64bd90b3548fbfd2791a36","observation_id":"d3fbf80f-7f20-4aa9-949e-88a247feca54","resolution":{"observed_at":"2026-08-15T22:00:10.303348Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.16130","last_updated":"2024-04-03T16:27:31Z","snapshot_observed_at":"2026-08-16T15:29:40.993745Z","submitted_at":"2023-05-25T15:04:01Z","title":"Language Models Implement Simple Word2Vec-style Vector Arithmetic","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.16130","snapshot_observed_at":"2026-08-15T22:00:10.309873Z","title":"Language models implement simple word2vec-style vector arithmetic","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.309873Z"},"links":{"cited_paper":"/paper/2305.16130","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:0d8be935e009762bba4284d83cede41fef56489e962d0946ba18e093655def1c","observation_id":"987d4032-2c32-437a-b2bd-6c8a72860bb0","resolution":{"observed_at":"2026-08-15T22:00:10.309873Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.322626Z","title":"In-context learning generalizes, but not always robustly: The case of syntax","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.322626Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:8013d3ee53e59639eab798333b33cd78f671111b8da66bc84ea0e69e6c9acf8f","observation_id":"2681c611-dad7-4fec-a5f4-722a252c47b6","resolution":{"observed_at":"2026-08-15T22:00:10.322626Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15852","last_updated":"2024-03-15T21:04:34Z","snapshot_observed_at":"2026-08-16T15:29:49.848201Z","submitted_at":"2023-05-25T08:43:46Z","title":"Self-contradictory Hallucinations of Large Language Models: Evaluation, Detection and Mitigation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15852","snapshot_observed_at":"2026-08-15T22:00:10.328149Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.328149Z"},"links":{"cited_paper":"/paper/2305.15852","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f80e0a7566a7b83f75164c73922cef1d73cc234b4f8bc2b4ccd0123a06f66cff","observation_id":"cff4232b-5ded-4a76-84dd-241b01e606b0","resolution":{"observed_at":"2026-08-15T22:00:10.328149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.333342Z","title":"Know what you don't know: Unanswerable questions for squad","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.333342Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:a82965998a765a0b363943d6f2b62814d3bdf11db0a7f7920d0262da7329d29a","observation_id":"67ed016a-5efd-4da6-bf20-2aa0df251489","resolution":{"observed_at":"2026-08-15T22:00:10.333342Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.00699","last_updated":"2025-07-09T19:56:02Z","snapshot_observed_at":"2026-08-16T14:04:51.278555Z","submitted_at":"2024-03-31T14:32:02Z","title":"A Comprehensive Survey of Contamination Detection Methods in Large Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.00699","snapshot_observed_at":"2026-08-15T22:00:10.339578Z","title":"How much are llms contaminated? A comprehensive survey and the llmsanitize library","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.339578Z"},"links":{"cited_paper":"/paper/2404.00699","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:72d369e836e28a6ad43157836ecb5e82d42e38bfe1f221d1056be91c9733731c","observation_id":"a9af2165-f0c5-4b2d-b9ec-99c587bbab03","resolution":{"observed_at":"2026-08-15T22:00:10.339578Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.345705Z","title":"Prompt programming for large language models: Beyond the few-shot paradigm","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.345705Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:46224600bb5a1f939f99ba583d3552e6256d30922ec309858e95dbbbbaa97a3f","observation_id":"29ced63f-40b0-47cb-b9c7-00ca0ccff7b5","resolution":{"observed_at":"2026-08-15T22:00:10.345705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.170522Z","title":"A natural experiment on LLM data contamination in code generation","venue":null,"work_id":"552f47a3-d880-449f-a309-b37c7bee3c70","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.353768Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:570e3137e5b27c584f6d73082f10564f07da37938f21f50e29e0ac36b88a838b","observation_id":"42649257-1eac-41fb-8027-bb5b2e74d59e","resolution":{"observed_at":"2026-08-15T22:00:11.174912Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.358528Z","title":"NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.358528Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:ed70d44e5c826c74788a3f2ed749216e4bb22324c9a18686c554a0d5504a377f","observation_id":"efe9794c-e9ff-4b6f-9d06-2e5f1528e393","resolution":{"observed_at":"2026-08-15T22:00:10.358528Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.153706Z","title":"Language models are greedy reasoners: A systematic formal analysis of chain-of-thought","venue":null,"work_id":"b36df335-9308-4a6e-ba35-8d24ffcfb7ae","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.364032Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:26614cc2aee4b403849679a46d4c7b36227578cf141852421c7abfc7ce9791bb","observation_id":"1905e593-47d4-498b-b7b1-f4089f73099e","resolution":{"observed_at":"2026-08-15T22:00:11.157774Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.140076Z","title":null,"venue":null,"work_id":"99966c14-1436-44d4-bcee-b6e50bc92cde","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.369363Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f92747dede2631c6d5102b13d9ed0fc802287a1535326507442224c5e85565b7","observation_id":"a9aff1ad-87af-4268-86ff-04027412bfba","resolution":{"observed_at":"2026-08-15T22:00:11.145225Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.10783","last_updated":"2026-08-05T15:52:23Z","snapshot_observed_at":"2026-08-10T02:39:33.166401Z","submitted_at":"2024-10-14T17:51:23Z","title":"LiveXiv -- A Multi-Modal Live Benchmark Based on Arxiv Papers Content","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.10783","snapshot_observed_at":"2026-08-15T22:00:10.374324Z","title":"Livexiv - A multi-modal live benchmark based on arxiv papers content","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.374324Z"},"links":{"cited_paper":"/paper/2410.10783","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:0c0831f69f26cdca46b6b4dbbda239d4746c900b6aff2b663d5a8b5e4e130ca8","observation_id":"d874dcdd-8913-499f-9040-60253a9f8e25","resolution":{"observed_at":"2026-08-15T22:00:10.374324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.380705Z","title":"Language models are multilingual chain-of-thought reasoners","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.380705Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:8f1dea576681b285fb2af0f1fdbe26bb591f97444b00549a1233290a4d164720","observation_id":"206f4690-61c5-4bef-b395-d58e57f13858","resolution":{"observed_at":"2026-08-15T22:00:10.380705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.03314","last_updated":"2024-08-06T17:35:05Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-08-06T17:35:05Z","title":"Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.03314","snapshot_observed_at":"2026-08-15T22:00:10.385552Z","title":"Scaling LLM test-time compute optimally can be more effective than scaling model parameters","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.385552Z"},"links":{"cited_paper":"/paper/2408.03314","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f7b6e947884e3b6ba58185a57434f36e11df4b5f2dbf440ab5a02359e9e48614","observation_id":"8b42a115-268b-457f-8442-b96fa8ed7686","resolution":{"observed_at":"2026-08-15T22:00:10.385552Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.112336Z","title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models","venue":null,"work_id":"e24e1e35-6cde-4dd2-92c2-dbfcc3c6c33a","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.390704Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:962f5b628851209201c44b30da1230e05eb69eaf93f23dd822dc6cc7efccf35d","observation_id":"4cd750b4-6f2e-423d-afa2-42bc77824238","resolution":{"observed_at":"2026-08-15T22:00:11.117221Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.nlrse-1.8","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.613526Z","title":null,"venue":null,"work_id":"93e40d9e-915d-44f3-80db-6d3ed710a62d","year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.395927Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:891e84522328d67d93d85ef8037445b2a1ee6b02b8fccf5b528cb1717ae6e2ee","observation_id":"5405adbb-fdac-4e6b-98d9-d107edeadaeb","resolution":{"observed_at":"2026-08-15T22:00:10.618573Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.093850Z","title":null,"venue":null,"work_id":"c587e40a-56a7-489e-8582-c6aa76cd5cf1","year":2019},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.401979Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:f9e12c41fa6299b746209eafbc7e4d0966e418eb834cd5885217e45cae06ffd1","observation_id":"8b42b87c-fe06-4142-b93a-bcc0cc850ca2","resolution":{"observed_at":"2026-08-15T22:00:11.100511Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2301.11916","last_updated":"2024-02-12T23:09:40Z","snapshot_observed_at":"2026-08-16T15:59:38.246095Z","submitted_at":"2023-01-27T18:59:01Z","title":"Large Language Models Are Latent Variable Models: Explaining and Finding Good Demonstrations for In-Context Learning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.11916","snapshot_observed_at":"2026-08-15T22:00:10.410510Z","title":"Large language models are implicitly topic models: Explaining and finding good demonstrations for in-context learning","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.410510Z"},"links":{"cited_paper":"/paper/2301.11916","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:6db8992e501d406526697264d7ce37a05dbeceabd53bf0c4f559856f7cb10881","observation_id":"3d52a43f-9462-4e91-838d-1191469dd8d1","resolution":{"observed_at":"2026-08-15T22:00:10.410510Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:10.419376Z","title":"Emergent analogical reasoning in large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.419376Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:4cf34e7603b37d06e41710d70b35be65f51219b40ec8ca79ea0e7851bf7bac52","observation_id":"62cfa28b-8cf7-406a-ae08-9ff25a4369f9","resolution":{"observed_at":"2026-08-15T22:00:10.419376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.066182Z","title":"Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, and William Fedus","venue":null,"work_id":"72eb9692-01f7-42d1-a77b-b1949c71d05b","year":2022},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.423734Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:96b09e6844bc9284021d06e1d618380101f24650788977a8fe7433587a00a239","observation_id":"1b51b3e6-bd18-48c8-809c-7c2f0eb35081","resolution":{"observed_at":"2026-08-15T22:00:11.071915Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.051089Z","title":"Chi, Quoc V","venue":null,"work_id":"273c96a5-b0c3-4fa2-8ad7-9cc59bc98507","year":2022},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.428533Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:d69792dffe25820fcd3c13588c753116c558dbf5bc9e7cf05c58397d837e8aab","observation_id":"b6c125aa-dd19-4f26-bcaa-b743fbf0b004","resolution":{"observed_at":"2026-08-15T22:00:11.056355Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19314","last_updated":"2025-04-18T19:36:00Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-06-27T16:47:42Z","title":"LiveBench: A Challenging, Contamination-Limited LLM Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19314","snapshot_observed_at":"2026-08-15T22:00:10.433948Z","title":"Livebench: A challenging, contamination-free LLM benchmark","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.433948Z"},"links":{"cited_paper":"/paper/2406.19314","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:9fadfef15bd4f42a24a24b90605985f4bf0253b2a8e32489d9c36d5a3878839c","observation_id":"1e998387-12eb-473e-b624-66cee9e76a4d","resolution":{"observed_at":"2026-08-15T22:00:10.433948Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T22:00:11.031755Z","title":"Adaptive chameleon or stubborn sloth: Revealing the behavior of large language models in knowledge conflicts","venue":null,"work_id":"5028f813-bda5-411c-a05e-c91986e69676","year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.443014Z"},"links":{"citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:33e6295a57be24271bc68cb0a23d8aa3634e892497abde2150182975e65c8c0b","observation_id":"b5a4af3f-3c19-4c74-80d1-433faf371b24","resolution":{"observed_at":"2026-08-15T22:00:11.038296Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-08-17T18:50:07.059564Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-15T22:00:10.450042Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.450042Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:588ec169d5054d5b02e04cb1ebaf3ff788f997fb4caf69db59f398c61cfa9102","observation_id":"7c7ec1f2-e068-407e-8440-e7518e97d44d","resolution":{"observed_at":"2026-08-15T22:00:10.450042Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.00332","last_updated":"2024-11-22T22:27:49Z","snapshot_observed_at":"2026-08-17T05:00:40.508036Z","submitted_at":"2024-05-01T05:52:05Z","title":"A Careful Examination of Large Language Model Performance on Grade School Arithmetic","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.00332","snapshot_observed_at":"2026-08-15T22:00:10.454387Z","title":"A careful examination of large language model performance on grade school arithmetic","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.454387Z"},"links":{"cited_paper":"/paper/2405.00332","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:d8bc02e9d2aea3da08bbdcd724dced8a348e15de5fc47c513ded113d40be39b0","observation_id":"95ca64ea-383a-474a-a83e-d399c0dea817","resolution":{"observed_at":"2026-08-15T22:00:10.454387Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.13534","last_updated":"2023-05-22T23:14:28Z","snapshot_observed_at":"2026-08-16T15:30:53.024861Z","submitted_at":"2023-05-22T23:14:28Z","title":"How Language Model Hallucinations Can Snowball","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.13534","snapshot_observed_at":"2026-08-15T22:00:10.460187Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.460187Z"},"links":{"cited_paper":"/paper/2305.13534","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:56694c12e0fbbc62c4eb6d3ed230bd55f70dc8db3e96c425beed842d2e756e7a","observation_id":"d833ab05-622d-4d38-8aed-abd04f39f60f","resolution":{"observed_at":"2026-08-15T22:00:10.460187Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.09927","last_updated":"2023-10-19T20:31:32Z","snapshot_observed_at":"2026-08-16T15:23:17.868564Z","submitted_at":"2023-06-16T15:50:03Z","title":"Trained Transformers Learn Linear Models In-Context","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.09927","snapshot_observed_at":"2026-08-15T22:00:10.464998Z","title":"Bartlett","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.464998Z"},"links":{"cited_paper":"/paper/2306.09927","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:d18028166c95f7875559ecb691b85903dccd86730aaa6c6dc235aeb201fa7d8a","observation_id":"0c9024f9-24d6-4bf0-b6c3-8fde42d56ac7","resolution":{"observed_at":"2026-08-15T22:00:10.464998Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.10513","last_updated":"2023-12-03T23:01:19Z","snapshot_observed_at":"2026-08-16T15:38:54.704364Z","submitted_at":"2023-04-20T17:48:43Z","title":"Why Does ChatGPT Fall Short in Providing Truthful Answers?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.10513","snapshot_observed_at":"2026-08-15T22:00:10.470063Z","title":"Why does chatgpt fall short in answering questions faithfully? CoRR, abs/2304.10513, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-15T22:00:10.470063Z"},"links":{"cited_paper":"/paper/2304.10513","citing_paper":"/paper/2505.08389"},"observation_digest":"sha256:e1b4632f31de697f5064c305216c22ccff32598d4770479290b59eed2cd92c13","observation_id":"ff9803c0-7e34-4f03-955c-2ea6bd322445","resolution":{"observed_at":"2026-08-15T22:00:10.470063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.08389","last_updated":"2025-05-13T09:35:40Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-17T11:00:28.602001Z","submitted_at":"2025-05-13T09:35:40Z","title":"Towards Contamination Resistant Benchmarks"},"reference_resolution":{"displayed":56,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":39,"verified_exact":2,"verified_fuzzy":15},"total_outbound_references":56},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 56 of 56 outbound references and 1 inbound Pith citation observation for arXiv:2505.08389."}