{"as_of":"2026-08-08T03:34:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:dc9bb70692f82810ba8648f3a2b08d5998c6881b650fafbfc52288cca277a16c","coverage":[{"denominator":37,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":37,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T12:35:29.070019Z","state":"measured"},{"denominator":37,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":37,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2505.24263/citation-record","integrity":"/paper/2505.24263/integrity","json":"/paper/2505.24263/citation-record.json","paper":"/paper/2505.24263"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.126963Z","title":"URL: \" 'urlintro :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.126963Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:5685b12a6d67df52447ba876673d717e21310f845a653fc13b514d92ec4ce29e","observation_id":"48403ae3-c436-4c4d-8b1b-b0882126a52f","resolution":{"observed_at":"2026-08-07T12:35:25.126963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.285963Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.285963Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:05fbf39746f4b9d49df39e9bbf41c98717dd21e3089ff448b5bcdfc5909f4a60","observation_id":"552d78f0-3bb1-4696-8fe9-f0b5a18c4c1a","resolution":{"observed_at":"2026-08-07T12:35:25.285963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.410371Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.410371Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:fe576321ac3acace529c6ed19665fd439beee870df7b166d1712143d1514c921","observation_id":"381ca8e0-ebc8-45b7-beb2-c466d9774410","resolution":{"observed_at":"2026-08-07T12:35:25.410371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:30.129677Z","title":null,"venue":null,"work_id":"395edd94-e160-4ca8-9358-5fbc14c79fd1","year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.533380Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:0d53d2b071069d7f16457c7f0adb4e6301645a1bcedfd859d563dc472d78c1ef","observation_id":"5d579d1d-a646-4908-bd15-bd401705277f","resolution":{"observed_at":"2026-08-07T12:35:30.205471Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2005.14165","last_updated":"2020-07-22T19:47:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-28T17:29:03Z","title":"Language Models are Few-Shot Learners","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.14165","snapshot_observed_at":"2026-08-07T12:35:25.691779Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.691779Z"},"links":{"cited_paper":"/paper/2005.14165","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:09e7a5787828a6876f8cfc5df07017908e11497b2b641b72b51996a5cfb757bc","observation_id":"ac6a04c4-0f96-44ff-a4d0-5d758d64bf20","resolution":{"observed_at":"2026-08-07T12:35:25.691779Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.07646","last_updated":"2023-03-06T06:28:18Z","snapshot_observed_at":"2026-08-03T19:51:05.627063Z","submitted_at":"2022-02-15T18:48:31Z","title":"Quantifying Memorization Across Neural Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.07646","snapshot_observed_at":"2026-08-07T12:35:25.837301Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.837301Z"},"links":{"cited_paper":"/paper/2202.07646","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:76562d41fa01a61a5bd1f8f379c1c1feba7b07bb507985294b94f1601fd5204d","observation_id":"7d2be1ac-92ae-4e22-9908-3792e513e538","resolution":{"observed_at":"2026-08-07T12:35:25.837301Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.945153Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.945153Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:c4a1d70e99bcbed88c6ff154ce34545cb5a14092396088364023a6a1f4419d85","observation_id":"1faed743-79ba-435e-a9d9-a5ba3cb31840","resolution":{"observed_at":"2026-08-07T12:35:25.945153Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-07T12:35:26.047017Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.047017Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:95ea6089ad1d1d8db09250202984b1dab5b1ad4a30a86407d242f692f8044572","observation_id":"6fb46aeb-d547-4ee8-8969-fc01656f5fca","resolution":{"observed_at":"2026-08-07T12:35:26.047017Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.18966","last_updated":"2025-05-09T23:13:37Z","snapshot_observed_at":"2026-07-06T19:39:15.025945Z","submitted_at":"2024-10-24T17:58:22Z","title":"Does Data Contamination Detection Work (Well) for LLMs? A Survey and Evaluation on Detection Assumptions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.18966","snapshot_observed_at":"2026-08-07T12:35:26.123753Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.123753Z"},"links":{"cited_paper":"/paper/2410.18966","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:ebb9043c1b275212745c26a2a33039869d99b7f419aff6b3caa236493f6f8c7c","observation_id":"d447cf9b-c72d-4059-a0f0-3344f8a8dc83","resolution":{"observed_at":"2026-08-07T12:35:26.123753Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04127","last_updated":"2025-01-10T14:31:21Z","snapshot_observed_at":"2026-08-02T00:54:17.243656Z","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04127","snapshot_observed_at":"2026-08-07T12:35:26.208319Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.208319Z"},"links":{"cited_paper":"/paper/2406.04127","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:c39e3261cfabf1c304b89d78ba42db0f8846d08e5342bc414fb96265834d1b3c","observation_id":"9b4366e9-05f8-49cd-a94a-145685687d6d","resolution":{"observed_at":"2026-08-07T12:35:26.208319Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08493","last_updated":"2024-02-21T22:02:26Z","snapshot_observed_at":"2026-08-06T07:48:14.479004Z","submitted_at":"2023-08-16T16:48:57Z","title":"Time Travel in LLMs: Tracing Data Contamination in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08493","snapshot_observed_at":"2026-08-07T12:35:26.309692Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.309692Z"},"links":{"cited_paper":"/paper/2308.08493","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:f44bf96f396daa5a862ee91bee0cc70111fb4d9bea2c65c29a2339b90d5f6dd1","observation_id":"65d8c614-fee6-4973-98cc-704b3e9632b4","resolution":{"observed_at":"2026-08-07T12:35:26.309692Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-07-06T18:55:11.576666Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-07T12:35:26.414783Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.414783Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:a7a71cb79ba2273648d802475fed52bd71ec0ed384e001ea9bfbf473c53a0219","observation_id":"4f148745-1121-4a19-8c0b-296d37a97674","resolution":{"observed_at":"2026-08-07T12:35:26.414783Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:26.515744Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.515744Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:fa929e656c2dbba1de3f9ec30f1bec75fbfe328f4e06c274654160dc36bb245c","observation_id":"f015e85b-9744-4278-af9d-5a08ebfe5715","resolution":{"observed_at":"2026-08-07T12:35:26.515744Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-07T12:35:26.585502Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.585502Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:e2c82492ff9d500a0d996958856fc6a845070f627ca978bf090331f77c69c3e6","observation_id":"f872dafe-c61b-4cd0-b962-c71c5ba91f46","resolution":{"observed_at":"2026-08-07T12:35:26.585502Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:29.959788Z","title":null,"venue":null,"work_id":"b957eb66-990d-45ca-a47c-09218f629875","year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.693692Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:14dc326a645e3b101cc2fddc6dac7d1d85b815a2e3eeba30256dcae944dffbf0","observation_id":"9fa753c5-b81e-466b-9385-c969a3c49ecf","resolution":{"observed_at":"2026-08-07T12:35:30.011952Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2106.09685","last_updated":"2021-10-16T18:40:34Z","snapshot_observed_at":"2026-08-07T07:43:16.294957Z","submitted_at":"2021-06-17T17:37:18Z","title":"LoRA: Low-Rank Adaptation of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.09685","snapshot_observed_at":"2026-08-07T12:35:26.812818Z","title":"Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.812818Z"},"links":{"cited_paper":"/paper/2106.09685","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:b3bdd637f5b16ddaacad2884f22026d04ac73f28f2c47479c1ded0a9b5510ef2","observation_id":"f23ba1cc-33d7-4198-8548-4725c998a439","resolution":{"observed_at":"2026-08-07T12:35:26.812818Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.07853","last_updated":"2022-02-03T04:29:23Z","snapshot_observed_at":"2026-08-06T13:05:17.451329Z","submitted_at":"2021-03-14T06:10:47Z","title":"Membership Inference Attacks on Machine Learning: A Survey","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.07853","snapshot_observed_at":"2026-08-07T12:35:26.911572Z","title":"Yu, and Xuyun Zhang","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.911572Z"},"links":{"cited_paper":"/paper/2103.07853","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:b55c6770fc10a836718b930f1862b6d9fa0a52b04ad1b1014f7eb66fba1ddd9b","observation_id":"088ddac3-4d2c-4796-97d3-8db2db9abe7b","resolution":{"observed_at":"2026-08-07T12:35:26.911572Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-07-06T16:20:36.866744Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-07T12:35:26.977795Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.977795Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:fd6c851d082d81675bbcfd22acfce475d3dcb1c7d991070f861ccc12ef995e28","observation_id":"3a55477a-f0cc-4719-8e26-4ed2979c2896","resolution":{"observed_at":"2026-08-07T12:35:26.977795Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.076291Z","title":null,"venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.076291Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:b4fe260028307b46953d1266b5505107a9829af38bf16d1a6281d1594f5f115e","observation_id":"c0af4880-be2f-4f7f-b237-dd4aed103c85","resolution":{"observed_at":"2026-08-07T12:35:27.076291Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:29.730038Z","title":"Lin, Jacob Hilton, and Owain Evans","venue":null,"work_id":"34a24a37-5e90-4ed9-8580-aedf62e6b92e","year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.190607Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:008a6da91012c16777a02dc12019226812a2226fcaa4cbda707e90078a4dc742","observation_id":"aca29502-3b06-4411-898b-37b2eb2c6e40","resolution":{"observed_at":"2026-08-07T12:35:29.821277Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.19437","last_updated":"2025-02-18T17:26:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-27T04:03:16Z","title":"DeepSeek-V3 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.19437","snapshot_observed_at":"2026-08-07T12:35:27.321284Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.321284Z"},"links":{"cited_paper":"/paper/2412.19437","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:30e544546c2e2f19b0553d4719800908f5ccd157d1b52195418a6014ba460c7a","observation_id":"e29cbdeb-17ed-44ca-be0f-87b2bd22759a","resolution":{"observed_at":"2026-08-07T12:35:27.321284Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.401062Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.401062Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:d7e98497fbf048ed20bca439b58efd851d8617c2d1f4dcef405a5dbd4b569753","observation_id":"537db04c-fc95-4991-b246-5bc604ff91bc","resolution":{"observed_at":"2026-08-07T12:35:27.401062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.01790","last_updated":"2025-02-28T02:40:58Z","snapshot_observed_at":"2026-08-07T05:21:59.292674Z","submitted_at":"2024-09-03T11:09:44Z","title":"Training on the Benchmark Is Not All You Need","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.01790","snapshot_observed_at":"2026-08-07T12:35:27.501910Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.501910Z"},"links":{"cited_paper":"/paper/2409.01790","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:6270705e8fc1e26428fa07258fb38453275468634a61db8859ffabf99c86896e","observation_id":"b4c6546f-481d-4c0f-a8a6-1c9b078a59b3","resolution":{"observed_at":"2026-08-07T12:35:27.501910Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T12:35:27.657313Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.657313Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:63b01487e62848184ff6090c491203cf751ff100542cd3815301b8e255e17ec4","observation_id":"022bc4e4-6bca-4107-a61d-e4d2afd41e0d","resolution":{"observed_at":"2026-08-07T12:35:27.657313Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.763705Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.763705Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:e1e843a0bb3d5d7f1f437a2cad7834d9579f1c1a73aa46adb7a663a38cb4f849","observation_id":"85948a63-eefd-4c53-8708-a7d5933f2fbb","resolution":{"observed_at":"2026-08-07T12:35:27.763705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-07T12:35:27.876148Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.876148Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:7ca7abeb056279c68f46ffa34e96a037253ff7411e5aabcc1db71342aa177eb0","observation_id":"457c0c24-c0e0-4a64-bfc4-7e22fc3024c4","resolution":{"observed_at":"2026-08-07T12:35:27.876148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.12353","last_updated":"2023-03-17T00:52:56Z","snapshot_observed_at":"2026-07-06T14:09:01.413494Z","submitted_at":"2022-10-22T05:04:54Z","title":"Leveraging Large Language Models for Multiple Choice Question Answering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.12353","snapshot_observed_at":"2026-08-07T12:35:27.989088Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.989088Z"},"links":{"cited_paper":"/paper/2210.12353","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:8a87114313d1cde76a6229ab96f4851bfb39b8f9c2e4ce7bdf6dcec10cba13ac","observation_id":"c8d45bf4-cb03-4b7e-b8af-fae0dddd9161","resolution":{"observed_at":"2026-08-07T12:35:27.989088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.09927","last_updated":"2024-12-08T19:15:26Z","snapshot_observed_at":"2026-08-03T19:23:12.561275Z","submitted_at":"2024-09-16T02:04:33Z","title":"Towards Data Contamination Detection for Modern Large Language Models: Limitations, Inconsistencies, and Oracle Challenges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.09927","snapshot_observed_at":"2026-08-07T12:35:28.145036Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.145036Z"},"links":{"cited_paper":"/paper/2409.09927","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:4f0fc9d87bda9a50a2fbbe44936aae3554aacd198f727aa7480efddd1c605fad","observation_id":"a1645894-ad5c-4459-87c3-38c1f24c1685","resolution":{"observed_at":"2026-08-07T12:35:28.145036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1904.09728","last_updated":"2019-09-09T17:29:55Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2019-04-22T05:36:37Z","title":"SocialIQA: Commonsense Reasoning about Social Interactions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.09728","snapshot_observed_at":"2026-08-07T12:35:28.259215Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.259215Z"},"links":{"cited_paper":"/paper/1904.09728","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:5aeae3ea67cc2fec5105f90180908b9af3067125a7c048ced058e3f20319f97b","observation_id":"d4a2cc5c-414e-4f72-93f1-84b947cfecfd","resolution":{"observed_at":"2026-08-07T12:35:28.259215Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-07T12:35:28.402129Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.402129Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:6c619d59210064248af7379c2581ce8d55e040d508fbca3a8fea5f93b1a2604d","observation_id":"877f8036-0eb7-428d-b4b5-b2904ec2975f","resolution":{"observed_at":"2026-08-07T12:35:28.402129Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08295","last_updated":"2024-04-16T12:52:47Z","snapshot_observed_at":"2026-08-03T03:29:01.959523Z","submitted_at":"2024-03-13T06:59:16Z","title":"Gemma: Open Models Based on Gemini Research and Technology","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08295","snapshot_observed_at":"2026-08-07T12:35:28.488255Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.488255Z"},"links":{"cited_paper":"/paper/2403.08295","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:dbcd9ec3cba41a758077f63364c759ad87e54a4c0dceb1ac4b7b8566ce6e3a7e","observation_id":"17509d7d-145e-4f20-b4c0-cafb062a0914","resolution":{"observed_at":"2026-08-07T12:35:28.488255Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-07T12:35:28.599455Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.599455Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:ce821d8d143073cad41abb34e9fbae10fc7f7dce4180f584e6ff79a1f28a85df","observation_id":"c169a4fc-480e-431e-9945-25576704a8e1","resolution":{"observed_at":"2026-08-07T12:35:28.599455Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-07T12:35:28.705768Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.705768Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:6633f7800b069e113e981124b97a61b468609603611a0d5fc8e5b0e08a13531e","observation_id":"d931bec7-4dd3-40f8-b778-8aef1d04f872","resolution":{"observed_at":"2026-08-07T12:35:28.705768Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01574","last_updated":"2024-11-06T02:54:00Z","snapshot_observed_at":"2026-08-06T00:29:17.674418Z","submitted_at":"2024-06-03T17:53:00Z","title":"MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01574","snapshot_observed_at":"2026-08-07T12:35:28.812524Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.812524Z"},"links":{"cited_paper":"/paper/2406.01574","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:aa152279eb77767be12c1b2587543fa3d29158a01f782e7d5c5be28dc3efdd1e","observation_id":"7265672e-7d0f-4ee7-aab8-3d9c9f00974d","resolution":{"observed_at":"2026-08-07T12:35:28.812524Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1910.03771","last_updated":"2020-07-14T03:42:34Z","snapshot_observed_at":"2026-07-06T08:27:58.343233Z","submitted_at":"2019-10-09T03:23:22Z","title":"HuggingFace's Transformers: State-of-the-art Natural Language Processing","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.03771","snapshot_observed_at":"2026-08-07T12:35:28.896218Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.896218Z"},"links":{"cited_paper":"/paper/1910.03771","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:a69b1f85bae2b0e70b063b186e2f4a0572b7bc63f61ea8c05e684f91c1af4122","observation_id":"cc895bca-1fb5-4f50-872f-432387d4c5bd","resolution":{"observed_at":"2026-08-07T12:35:28.896218Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18824","last_updated":"2024-04-29T16:05:36Z","snapshot_observed_at":"2026-07-06T18:07:10.639325Z","submitted_at":"2024-04-29T16:05:36Z","title":"Benchmarking Benchmark Leakage in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18824","snapshot_observed_at":"2026-08-07T12:35:28.982799Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.982799Z"},"links":{"cited_paper":"/paper/2404.18824","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:193c6fcbeff8a5c251e7b569db1273fa756809f801d60289bfa68f45faff6df0","observation_id":"cbd67ca9-a106-43a1-a3b5-d6a351c50b4f","resolution":{"observed_at":"2026-08-07T12:35:28.982799Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1905.07830","last_updated":"2019-05-19T23:57:23Z","snapshot_observed_at":"2026-07-31T00:09:56.948833Z","submitted_at":"2019-05-19T23:57:23Z","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.07830","snapshot_observed_at":"2026-08-07T12:35:29.070019Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:29.070019Z"},"links":{"cited_paper":"/paper/1905.07830","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:3e3fc830c2ae54c12e8bf0ad71a8c7b75013ed0e8fa5c748c0f56fdc053bb925","observation_id":"dfe21523-ee1b-44fc-b0db-954c381c2c42","resolution":{"observed_at":"2026-08-07T12:35:29.070019Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T12:25:04.084467Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation"},"reference_resolution":{"displayed":37,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":36,"verified_exact":0,"verified_fuzzy":1},"total_outbound_references":37},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 37 of 37 outbound references and 0 inbound Pith citation observations for arXiv:2505.24263."}