{"as_of":"2026-08-08T03:39:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:09045bc2ff87afa3b72bf10f01dfd585cca7cbf012d55cda420ad215cff757a5","coverage":[{"denominator":78,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":78,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T14:45:42.780856Z","state":"measured"},{"denominator":79,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":79,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T17:21:27.531024Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.18182","snapshot_observed_at":"2026-08-03T17:21:27.531024Z","title":"CoRR , volume =","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2512.11001","last_updated":"2026-07-07T14:01:07Z","snapshot_observed_at":"2026-08-07T02:51:28.184118Z","submitted_at":"2025-12-10T20:16:20Z","title":"Rethinking Query Optimization for Multi-Agent Systems [Vision]","version":2},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-03T17:21:27.531024Z"},"links":{"cited_paper":"/paper/2507.18182","citing_paper":"/paper/2512.11001"},"observation_digest":"sha256:896833d3d5acc8de6f5e26196b1792f80da79d28060c424726d80a5418ac49e2","observation_id":"0c7b0f32-2814-431a-9a24-54afd8f992c0","resolution":{"observed_at":"2026-08-03T17:21:27.531024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2507.18182/citation-record","integrity":"/paper/2507.18182/integrity","json":"/paper/2507.18182/citation-record.json","paper":"/paper/2507.18182"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-06T14:45:42.561961Z","title":"Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.561961Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:94f181de65417d213ad03a6779547e4d479355550eb6abed5fa2c3e21337f5d1","observation_id":"5216c661-9e79-4deb-9ee1-06cda3c739d5","resolution":{"observed_at":"2026-08-06T14:45:42.561961Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.12712","last_updated":"2023-04-13T20:41:31Z","snapshot_observed_at":"2026-08-03T04:49:15.195814Z","submitted_at":"2023-03-22T16:51:28Z","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.12712","snapshot_observed_at":"2026-08-06T14:45:42.566066Z","title":"Sparks of artificial general intelligence: Early experiments with gpt-4","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.566066Z"},"links":{"cited_paper":"/paper/2303.12712","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f3372b3ddc276c29dc93dbe5e3c5ac3d3dab5873b78323f39a97b1c40ee28126","observation_id":"cb2282c2-f384-4bd5-b127-3f706c3f7a62","resolution":{"observed_at":"2026-08-06T14:45:42.566066Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-06T14:45:42.569568Z","title":"Llama: Open and efficient foundation language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.569568Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:20a7f871331b432e2b715daf9a2515fa0ae2c60385d0890596a220aa4e4dc2b9","observation_id":"2760630c-638e-44ae-86a4-a0c55b30d2d3","resolution":{"observed_at":"2026-08-06T14:45:42.569568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-06T14:45:42.572487Z","title":"Gemini: a family of highly capable multimodal models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.572487Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:20eca2886285cf48781ddbf19cb1f13a863a915b871e93af56d04bd94d28f670","observation_id":"f9293f9f-fa3a-444a-b726-edeaf7c1a832","resolution":{"observed_at":"2026-08-06T14:45:42.572487Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.618524Z","title":"The economic potential of generative ai: The next productivity frontier, 2023","venue":null,"work_id":"26cec241-0f45-4c87-a546-19389588577e","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.575869Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:20a4536e997a9bac8b99bec32a4f1c5fdd62065becb24ba186a5481dc84cc15c","observation_id":"218de190-04dc-4364-95d0-850a2e06143d","resolution":{"observed_at":"2026-08-06T14:45:43.621636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.609493Z","title":"Pwc is accelerating adoption of ai with chatgpt enterprise in us and uk and with clients, 2024","venue":null,"work_id":"b9cc843b-c0dd-4e0e-ae33-52e533f19f01","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.579151Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:4fb5648d24a52b20d68d05b42fefc1f9f6f57832542496b3479e2f4d7f10dd20","observation_id":"a77f83e9-b076-47b2-bab1-54b48e044733","resolution":{"observed_at":"2026-08-06T14:45:43.612570Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.01869","last_updated":"2024-08-06T11:58:53Z","snapshot_observed_at":"2026-08-04T13:02:37.475538Z","submitted_at":"2024-04-02T11:46:31Z","title":"Beyond Accuracy: Evaluating the Reasoning Behavior of Large Language Models -- A Survey","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.01869","snapshot_observed_at":"2026-08-06T14:45:42.582894Z","title":"Beyond accuracy: evaluating the reasoning behavior of large language models–a survey","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.582894Z"},"links":{"cited_paper":"/paper/2404.01869","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:3b291cabac4cd9211502d61a3d6933f33fc209f517297430121b1975ada93d02","observation_id":"8d2d8d73-ee8e-4180-82a9-32fcbcc1dac6","resolution":{"observed_at":"2026-08-06T14:45:42.582894Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.599983Z","title":"Shortcut learning of large language models in natural language understanding","venue":null,"work_id":"44f72b25-531b-433d-9d88-6a5261b5a888","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.585882Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:29a55e15bd496278e3d8f63d10143a80a89c2ddcc6def6450b1c85352e5c1e7e","observation_id":"58eb326e-4fdb-4ed6-9560-5333f4283e41","resolution":{"observed_at":"2026-08-06T14:45:43.603251Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.03205","last_updated":"2025-05-30T20:24:51Z","snapshot_observed_at":"2026-08-07T10:57:34.323588Z","submitted_at":"2024-05-06T07:10:09Z","title":"Anchored Answers: Unravelling Positional Bias in GPT-2's Multiple-Choice Questions","version":3},"cited_work":{"arxiv_id":"2405.03205","doi":null,"metadata_source":"pith","pith_arxiv_id":"2405.03205","snapshot_observed_at":"2026-08-06T14:45:43.141576Z","title":"Anchored Answers: Unravelling Positional Bias in GPT-2's Multiple-Choice Questions","venue":"cs.CL","work_id":"8e2c79e6-a649-4b6c-a46d-005320e120a0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.591793Z"},"links":{"cited_paper":"/paper/2405.03205","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:bdfbf6c2fa22aae0a79fc076207e02865ab3ea8c2d6145165a4fd43e929aad6d","observation_id":"f349f8bc-052c-40fb-9d71-1c872ffe3904","resolution":{"observed_at":"2026-08-06T14:45:43.145432Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.15393","last_updated":"2024-10-20T13:47:39Z","snapshot_observed_at":"2026-08-04T14:47:05.097254Z","submitted_at":"2024-10-20T13:47:39Z","title":"CalibraEval: Calibrating Prediction Distribution to Mitigate Selection Bias in LLMs-as-Judges","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.15393","snapshot_observed_at":"2026-08-06T14:45:42.595463Z","title":"Calibraeval: Calibrating prediction distribution to mitigate selection bias in llms-as-judges","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.595463Z"},"links":{"cited_paper":"/paper/2410.15393","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c8a20ea376256b5dee2dba269f1898c2beb92e984e230db965cfd2d20cc1de9c","observation_id":"48e8a152-485c-49e4-9a07-f5963bc2dd57","resolution":{"observed_at":"2026-08-06T14:45:42.595463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08382","last_updated":"2024-08-20T08:07:49Z","snapshot_observed_at":"2026-08-05T05:16:04.368829Z","submitted_at":"2024-04-12T10:36:15Z","title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think","version":2},"cited_work":{"arxiv_id":"2404.08382","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.08382","snapshot_observed_at":"2026-08-06T14:45:43.121164Z","title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think","venue":"cs.CL","work_id":"f7af48c2-fb1d-4fff-939c-198e46fe32d9","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.598610Z"},"links":{"cited_paper":"/paper/2404.08382","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:b81451694be7a21c9e47dc05d8ac608eaf2546a6d562f1ca33de1bb98d6faadf","observation_id":"804d4f28-d333-42c6-9101-1f053eb2d790","resolution":{"observed_at":"2026-08-06T14:45:43.124439Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.601429Z","title":"Language models are few-shot learners","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.601429Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:3f6f7030bd47c38a1a19980202fb63f5448bc25b5a55c06b0d598dec99e8f5c7","observation_id":"e2ddc13a-a790-4983-8e6b-0df7fcbf2a43","resolution":{"observed_at":"2026-08-06T14:45:42.601429Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.604125Z","title":"Exploring the limits of transfer learning with a unified text-to-text transformer","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.604125Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:0fd7b2f345debd538b399cb115a7b235af5534a84664e71cdf562b35fc0dfff8","observation_id":"0cc113ea-39ac-4f19-a6ef-57a2f6c3590c","resolution":{"observed_at":"2026-08-06T14:45:42.604125Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-06T14:45:42.607027Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.607027Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2da3446535b33b98f4c9b583c96fc8c2f643ba0efbbfe961105f6df904c481e1","observation_id":"d5ea368c-46eb-4423-8f61-3f20b0bb8b88","resolution":{"observed_at":"2026-08-06T14:45:42.607027Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.578988Z","title":"Commonsenseqa: A question answering challenge targeting commonsense knowledge","venue":null,"work_id":"a3a89ee0-eae5-42f1-b6a6-910030603c84","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.609512Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:5714e2536e4b857ee42ea6aeb7d0772b075d0f069d28519c31d2df6974c570e2","observation_id":"792dc029-15b8-4a49-bf2f-6bb7d7223cfd","resolution":{"observed_at":"2026-08-06T14:45:43.581828Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-06T14:45:42.612530Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.612530Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:02fbffcfc5e612f849f82b1e50d9651588b0ec5522e5be737e700c174a966a24","observation_id":"ca26ebfc-5895-458f-80aa-cdd3cebe1982","resolution":{"observed_at":"2026-08-06T14:45:42.612530Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.615299Z","title":"Can a suit of armor conduct electricity? a new dataset for open book question answering","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.615299Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2f8a201a1d515117c0cca32f339fd341cb6a29dda47d7f8e3934b534cfee68e2","observation_id":"ad484efe-39c1-4c9f-9489-a11f91e62622","resolution":{"observed_at":"2026-08-06T14:45:42.615299Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.564500Z","title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models","venue":null,"work_id":"538e3469-9e3a-4b00-9f56-93b604fbd42b","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.618154Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f479a44fc0adac1f4333d2a17d15a09946c9d9c954ea7116ad6fc6666f212ffb","observation_id":"bd3fae3e-6895-46f2-8f8a-9d697081e08e","resolution":{"observed_at":"2026-08-06T14:45:43.567456Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-01T19:14:56.803459Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-06T14:45:42.620508Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.620508Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:96600e66771e37d8850b99123aa17f74a220b29c8a261ae77eb0c19c7c496342","observation_id":"b91a51a6-a339-4d1e-b70a-3fb91eafeb9b","resolution":{"observed_at":"2026-08-06T14:45:42.620508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.555280Z","title":"M3exam: A multilingual, multimodal, multilevel benchmark for examining large language models","venue":null,"work_id":"b8e6f6a4-e43b-49b6-9d87-b40b0f23ed6f","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.623159Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:1636ac59f110af480e09baafbae2597c79c77453a4e30d4ac285782c1ca44244","observation_id":"a999ef10-10e4-4015-be27-90b254a96c08","resolution":{"observed_at":"2026-08-06T14:45:43.558566Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.546941Z","title":"Agieval: A human-centric benchmark for evaluating foundation models","venue":null,"work_id":"c49e0475-f072-4e9b-b7e2-5d746de2ad93","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.625905Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:343379658a294bf3d82faf85c4ba10c5ac207085543a1c9369307405d8e4d352","observation_id":"8123bda6-f4b8-46fc-8f07-d6a3cbaaecc7","resolution":{"observed_at":"2026-08-06T14:45:43.549898Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.628168Z","title":"Medmcqa: A large-scale multi-subject multi-choice dataset for medical domain question answering","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.628168Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ed0fe23fe4defebc1f5839f6b48b484300105126598268ff47ff3d9993ea2755","observation_id":"d3e3c2aa-2ceb-4a65-8b2d-724c83c9f536","resolution":{"observed_at":"2026-08-06T14:45:42.628168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.630597Z","title":"Crowdsourcing multiple choice science questions","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.630597Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:e8e479bc2efe97b7fecf670f3a607b446322df6f375d5c6ee8d8f79f84e7d263","observation_id":"df6b1510-bd09-462b-9813-4e35ba20437d","resolution":{"observed_at":"2026-08-06T14:45:42.630597Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.526391Z","title":"From live data to high-quality benchmarks: The arena-hard pipeline","venue":null,"work_id":"d2cc26dd-a3df-4180-9301-9b6feeb22bdb","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.632984Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:e98ea4e528c8f78510d7a6ff5a28bdd028be671f18cdd9379ed46e34c675b178","observation_id":"0d09c583-8dd4-40cc-b490-2e0d5ca64a9d","resolution":{"observed_at":"2026-08-06T14:45:43.529542Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.635387Z","title":"Chatbot arena: An open platform for evaluating llms by human preference","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.635387Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:5e1a0c82c6a885ca0510235133ef1eea7842fd858f2903511c481a7cea9d2efb","observation_id":"e3db5bdc-e636-4f69-a407-95adab2f9f80","resolution":{"observed_at":"2026-08-06T14:45:42.635387Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.511515Z","title":"Large language models are not fair evaluators","venue":null,"work_id":"f01a0b4d-339b-43d5-a8d5-2487e630526a","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.638295Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:d3e4b866b1a9caaf9c735bd62f07c05cb1f9bbccfdacb2c082b981f452121f70","observation_id":"9f7a62ba-9fde-4987-ae00-0777e92a8a65","resolution":{"observed_at":"2026-08-06T14:45:43.514598Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.502649Z","title":"Apbench and benchmarking large language model performance in fundamental astrodynamics problems for space engineering","venue":null,"work_id":"30fa044b-d9f8-4706-b9de-a997593d90b1","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.640769Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:70d49f078cc9b35d3faf3512eb9ac5e8857062b9586110ee600a66756f7710a1","observation_id":"ab2c8cd0-06b0-4f5d-9b3f-6be3e265e722","resolution":{"observed_at":"2026-08-06T14:45:43.505622Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.491806Z","title":"Where is the answer? an empirical study of positional bias for parametric knowledge extraction in language model","venue":null,"work_id":"77e371de-f3b0-4f55-a699-30a6aa37ee82","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.643189Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:00d95b331747052355b36cb9cc83bac6598f99e61d0a8a3a04f1b6203f10037f","observation_id":"ecf966ea-e5af-4d75-a462-0c8c733f6360","resolution":{"observed_at":"2026-08-06T14:45:43.495745Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.482212Z","title":"Option symbol matters: Investigating and mitigating multiple-choice option symbol bias of large language models","venue":null,"work_id":"7180469c-5f47-4a21-8d0e-a84e0922d60f","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.645709Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:097a578ebea62f04a6a2c1380c488cff03e33d2018de05d6fe1f921eaff3bc14","observation_id":"64cad35f-936a-4df2-9b8a-c5f5aff12226","resolution":{"observed_at":"2026-08-06T14:45:43.485373Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.473409Z","title":"Large language models sensitivity to the order of options in multiple- choice questions","venue":null,"work_id":"2f0fce3e-c072-4e6c-ae37-0b260076ee19","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.648245Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:69631d45f4260b44ae236545dc7b85170cc1f318f81643472cbfa5d8d4146acb","observation_id":"0627c88b-3464-416e-bfce-57c47b93af5b","resolution":{"observed_at":"2026-08-06T14:45:43.476599Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.03882","last_updated":"2024-02-22T01:40:35Z","snapshot_observed_at":"2026-08-06T19:10:47.693909Z","submitted_at":"2023-09-07T17:44:56Z","title":"Large Language Models Are Not Robust Multiple Choice Selectors","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.03882","snapshot_observed_at":"2026-08-06T14:45:42.650856Z","title":"Large language models are not robust multiple choice selectors","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.650856Z"},"links":{"cited_paper":"/paper/2309.03882","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f32877603c874806585c55c1a2ead6f4526ee2fa53f283ed6b07bb41e02cc279","observation_id":"23bf2d76-da00-464e-8b44-1cf3feee1a49","resolution":{"observed_at":"2026-08-06T14:45:42.650856Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.464099Z","title":"Fool your (vision and) language model with embarrassingly simple permutations","venue":null,"work_id":"f526553c-6805-41ed-b934-7fb550133fb5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.653438Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:cf28d1a05e83e2a56d037d87ac543223c96efede207e5eaf311c293fb86fbd54","observation_id":"fee25609-a4fd-4c9e-b357-493252f824f9","resolution":{"observed_at":"2026-08-06T14:45:43.467133Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.454963Z","title":"Teacher-student training for debiasing: General permutation debiasing for large language models","venue":null,"work_id":"a7826d21-d4cc-46c1-98c0-2920a71344d3","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.656108Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9ea5f00c6cab401df6f551f3b154ac526cef8e72936d08ab267e6b11e9ddcda2","observation_id":"4fd8a892-a18c-4df6-acf8-52a6a94ba69c","resolution":{"observed_at":"2026-08-06T14:45:43.458257Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2409.18857","last_updated":"2025-05-17T04:21:30Z","snapshot_observed_at":"2026-07-06T19:23:29.050430Z","submitted_at":"2024-09-27T15:53:54Z","title":"Mitigating Selection Bias with Node Pruning and Auxiliary Options","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.18857","snapshot_observed_at":"2026-08-06T14:45:42.658846Z","title":"Mitigating selection bias with node pruning and auxiliary options","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.658846Z"},"links":{"cited_paper":"/paper/2409.18857","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9b7c2b8c6166c764aab0162446cfeceb400852e1520c1d4b7ded1625dc660f01","observation_id":"49d0fb8b-749a-4c49-a354-e988cd73e886","resolution":{"observed_at":"2026-08-06T14:45:42.658846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.445455Z","title":"Unveiling selection biases: Exploring order and token sensitivity in large language models","venue":null,"work_id":"a7abbaa0-c4bc-41fa-9085-c9a87c8aa3a0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.661589Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6db4859ff7e7ed90b0c017dd143612d1e5370c1833f841fbff1ed963b79fab0b","observation_id":"27c90e41-76f3-4519-8ba4-c8865cdb35ec","resolution":{"observed_at":"2026-08-06T14:45:43.448857Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.664558Z","title":"Chain-of-thought prompting elicits reasoning in large language models.Advances in neural information processing systems, 35:24824–24837, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.664558Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6031dec44bd7676f4b747d5074f4828405235825ad27379202b204311be30dd1","observation_id":"c5073e1d-af59-4fab-a0aa-60dbc4c9be1b","resolution":{"observed_at":"2026-08-06T14:45:42.664558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.667059Z","title":"Large language models are zero-shot reasoners","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.667059Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:7c59b6a837d815e6377625b062523bee787452b49cd7c1c83660ba4b07851628","observation_id":"c5c97aeb-d96c-41b5-b53c-15482b0c79ff","resolution":{"observed_at":"2026-08-06T14:45:42.667059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.11171","last_updated":"2023-03-07T17:57:37Z","snapshot_observed_at":"2026-07-06T12:50:22.773056Z","submitted_at":"2022-03-21T17:48:52Z","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.11171","snapshot_observed_at":"2026-08-06T14:45:42.669715Z","title":"Self-consistency improves chain of thought reasoning in language models.arXiv preprint arXiv:2203.11171, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.669715Z"},"links":{"cited_paper":"/paper/2203.11171","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:345714ccc992f4635cdce0d5bb3d349c8e13eff1f70453efbc6e8bf81c657275","observation_id":"7ed7787b-ed82-49ae-9e66-846a8d6348ed","resolution":{"observed_at":"2026-08-06T14:45:42.669715Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.424270Z","title":"Star: Self-taught reasoner bootstrapping reasoning with reasoning","venue":null,"work_id":"09ea7ace-09d5-4008-a05a-da4bc56b5b43","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.672549Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:d9cd3bd85986836ef11b8ebc4ee3ceb0405bbe3d059176b2072c3e9f3e80162b","observation_id":"5bbae5f3-6a6e-4388-ae67-6029fe8ad69a","resolution":{"observed_at":"2026-08-06T14:45:43.427766Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2205.10625","last_updated":"2023-04-16T22:08:08Z","snapshot_observed_at":"2026-08-06T09:00:42.886249Z","submitted_at":"2022-05-21T15:34:53Z","title":"Least-to-Most Prompting Enables Complex Reasoning in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.10625","snapshot_observed_at":"2026-08-06T14:45:42.676271Z","title":"Least-to-most prompting enables complex reasoning in large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.676271Z"},"links":{"cited_paper":"/paper/2205.10625","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9830df45fccebe69b63e0c138f8d2b9f5d2682821b533d912eb4b2a820d637c5","observation_id":"e06a5064-bf7a-4a76-b3ef-2bab70a4aafa","resolution":{"observed_at":"2026-08-06T14:45:42.676271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.680112Z","title":"React: Synergizing reasoning and acting in language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.680112Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:80044a9ffb1ebc430bd15032586c4345bb19c7cfde004bc3f9ab383879443f5e","observation_id":"7d3b7e95-806d-40bb-8b0a-78b2baf192ec","resolution":{"observed_at":"2026-08-06T14:45:42.680112Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.409524Z","title":"Debiasing in-context learning by instructing llms how to follow demonstrations","venue":null,"work_id":"0b3d6a8a-b63f-4e9f-8b97-efa13030a9c5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.683542Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:341101093dcb0200066590e5913c27d9b3414c829bdfbddb1d7fad638e600dee","observation_id":"9d7f7b60-f742-4ed4-8262-538aa0aa21a9","resolution":{"observed_at":"2026-08-06T14:45:43.412777Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.12588","last_updated":"2023-10-23T01:27:38Z","snapshot_observed_at":"2026-08-02T13:06:11.850456Z","submitted_at":"2022-11-22T21:06:00Z","title":"Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.12588","snapshot_observed_at":"2026-08-06T14:45:42.686346Z","title":"Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.686346Z"},"links":{"cited_paper":"/paper/2211.12588","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:01876550783c24234b92a7f1e2f87d6fd59c7472c5a785ef1d94d4678125ea96","observation_id":"568a757c-58f2-439f-9301-512f20c93783","resolution":{"observed_at":"2026-08-06T14:45:42.686346Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.400365Z","title":"Prompt sketching for large language models","venue":null,"work_id":"c882cf97-8b8d-4d70-98fc-07d04088d229","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.689179Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c145fd715193131497767475fe229daac6c4c20746747b645dfd01d5c0ba540f","observation_id":"46c97963-d8cb-490f-b595-6a688a93b16c","resolution":{"observed_at":"2026-08-06T14:45:43.403188Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.691864Z","title":"Self-refine: Iterative refinement with self-feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.691864Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9e7e5c2a3739f14a852815e1535ae15ab271dcec9434fe773afae25aaaa516bb","observation_id":"25ae7ea6-21e6-4729-a55d-f3fa6c05b86a","resolution":{"observed_at":"2026-08-06T14:45:42.691864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-06T14:45:42.694535Z","title":"Constitutional ai: Harmlessness from ai feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.694535Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:7d04fb269f3ca3aafdba22cc2e8ed2728c49cc23ad34e83cef10d42116811fd8","observation_id":"88a31bde-6223-435b-90a9-6c6349be6b86","resolution":{"observed_at":"2026-08-06T14:45:42.694535Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.386360Z","title":"Neurologic decoding:(un) supervised neural text generation with predicate logic constraints","venue":null,"work_id":"17436e4e-b7ac-4482-ba86-bc4535076acc","year":2021},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.697124Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:66839b9b9152e7ea79af1deb83c139d36c87bd18f4a440c979a24ce07e271114","observation_id":"74233023-f874-45d8-8741-9eebd5e2d606","resolution":{"observed_at":"2026-08-06T14:45:43.389274Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.376968Z","title":"Calibration of pre-trained transformers","venue":null,"work_id":"4eb64bff-b1ab-46f6-853c-19564353a8dc","year":2020},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.699431Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:90b6cba7cde06e078d60db69a76fb6afcefde215f26ed34263e796549122e873","observation_id":"fe1876f5-78d7-4d4c-b641-4d40858ffdf3","resolution":{"observed_at":"2026-08-06T14:45:43.379919Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.701730Z","title":"Calibrate before use: Improving few-shot performance of language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.701730Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:168457b7e7aab9e5844590d4d879944d6a6ea96b49136bbcc4517bcb7c3380a3","observation_id":"5ceeb87a-9260-4404-b871-4ed2589f7ac2","resolution":{"observed_at":"2026-08-06T14:45:42.701730Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.362088Z","title":"Calibrating language models with adaptive temperature scaling","venue":null,"work_id":"dc539f5f-9bcb-4f21-a55a-795a4af0b706","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.704480Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:20375bb8283f55df2848115306c24952ae990156b25f7cbdde330dea0736cc94","observation_id":"5f4750a8-0d4e-46bd-9a14-32a3ef571d81","resolution":{"observed_at":"2026-08-06T14:45:43.365364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.353566Z","title":"Calibrating large language models with sample consistency","venue":null,"work_id":"6372a039-fe73-42c4-8ec0-bf2f1b4f88a4","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.707052Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6e888125afb249a371d62de849ea5d3757efd71be29eec1c07184b3a81e5149e","observation_id":"291e6d06-dc20-49f7-91ee-40bff5409dc0","resolution":{"observed_at":"2026-08-06T14:45:43.356435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.344466Z","title":"Benchmarking uncertainty quantification methods for large language models with lm-polygraph","venue":null,"work_id":"3f9d4d4a-aeaa-41e9-b271-fa18dc13bd6b","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.709284Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:80b3d6555f9fbcf661885afa95696445a9a85e2100b170cf0f325db2dff125dd","observation_id":"f28128f1-d3a7-4447-b8ad-b0a4c509c41b","resolution":{"observed_at":"2026-08-06T14:45:43.347645Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.335449Z","title":"Thermometer: towards universal calibration for large language models","venue":null,"work_id":"6c2a6462-0269-44b0-8cb6-b6011eddebd3","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.711759Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:3934d95c8acda40cb33a07a18daf6732ad2d345a907cabd5ddbeea2ee26df618","observation_id":"c7ad5102-cabb-474c-b834-4c0c79213041","resolution":{"observed_at":"2026-08-06T14:45:43.338319Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.326679Z","title":"Monte carlo temperature: a robust sampling strategy for llm’s uncertainty quantification methods","venue":null,"work_id":"995fc3c7-98ac-4dae-865e-fdd67f5bd0a8","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.714104Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:02a665d6cdb4305041b5f470d46f24902e67250d0d97d5f328e60be68ea59776","observation_id":"981f2f23-405a-4692-a00e-7f33bd4543e3","resolution":{"observed_at":"2026-08-06T14:45:43.329636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.716462Z","title":"Charm: Calibrating reward models with chatbot arena scores","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.716462Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:533206258dd37de5271d4731d5ebbc3b2673002a5f15d20e27d092ef46b7e925","observation_id":"30a565e2-2bc1-44e8-947c-a349b0dbe6b3","resolution":{"observed_at":"2026-08-06T14:45:42.716462Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.719088Z","title":"Restoring calibration for aligned large language models: A calibration-aware fine-tuning approach","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.719088Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:98ae23622977906d126db391fdcd497e9be9048fdf4feef67810e9689946f24f","observation_id":"8941c8ba-16f9-438b-9dcb-e1edbbcd7d68","resolution":{"observed_at":"2026-08-06T14:45:42.719088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.317814Z","title":"Uncertainty estimation in large language models to support biodiversity conservation","venue":null,"work_id":"9033ea43-e6ba-4f73-a35b-fc6af24e1820","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.721589Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:d018630f088df5dabc500afaf016b26d6cbf35124fc032a40cb8bf9efd5b93e7","observation_id":"54a0abea-7d07-4838-bc15-dd125385f050","resolution":{"observed_at":"2026-08-06T14:45:43.321254Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2302.02083","last_updated":"2024-11-04T19:51:53Z","snapshot_observed_at":"2026-08-07T09:50:45.770227Z","submitted_at":"2023-02-04T03:50:01Z","title":"Evaluating Large Language Models in Theory of Mind Tasks","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.02083","snapshot_observed_at":"2026-08-06T14:45:42.724363Z","title":"Theory of mind may have spontaneously emerged in large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.724363Z"},"links":{"cited_paper":"/paper/2302.02083","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f35d3157736038ec31b32d46ef77cf75d71b5810e5cd93641faeefa09ea01962","observation_id":"c4da330a-b34e-4de7-b732-7b850c1a77d2","resolution":{"observed_at":"2026-08-06T14:45:42.724363Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.309088Z","title":"Neural theory-of-mind? on the limits of social intelligence in large lms","venue":null,"work_id":"e168591c-c5af-4d54-8403-56f8ec19a568","year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.727107Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:cbfd954bd1cc8d933c26fa1f50f777afa7ae0228778d321b117317a8f7a75976","observation_id":"45e80a02-291f-45f8-af0f-f452ba01f3b9","resolution":{"observed_at":"2026-08-06T14:45:43.311892Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.300905Z","title":"Social iqa: Commonsense reasoning about social interactions","venue":null,"work_id":"d44cf160-8666-45a0-a063-a612a33dd624","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.729494Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:abe94abf8461b75e7361b40c53f5f3747c60c9334dc66109b1d535a921087948","observation_id":"3732c923-91aa-4618-9630-d810928a1e8b","resolution":{"observed_at":"2026-08-06T14:45:43.303710Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.291488Z","title":"Large language models are not strong abstract reasoners","venue":null,"work_id":"85db9d18-8640-4fe2-929b-59a6b63bfe9e","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.732181Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:77f00a2667bb7540cae53cf765075e53acc521700eafb779290412f999f1bb6c","observation_id":"f6ae2707-9ad2-4811-ade7-46f523ba93a3","resolution":{"observed_at":"2026-08-06T14:45:43.295120Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.283071Z","title":"Coglm: Tracking cognitive development of large language models","venue":null,"work_id":"da25771b-e8e2-42a5-b73d-369faad7fbb2","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.734531Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:08f787ac7599c042107ece4aef17aec218eb7709efc2889a39e7e73db850fcec","observation_id":"0c5f58cd-35f4-43ac-a61b-3680e204b522","resolution":{"observed_at":"2026-08-06T14:45:43.285900Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.275035Z","title":"V-alphasocial: Benchmark and self-reflective chain-of- thought generation for visual social commonsense reasoning","venue":null,"work_id":"0201843c-0e81-429f-bd89-0ea966e6cef2","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.736943Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:511efe50c41a2d057d2e641d15b9248394e9a543a77645bcd96a988fae1698ab","observation_id":"51baff84-0779-4c6e-9fb3-16938bc28486","resolution":{"observed_at":"2026-08-06T14:45:43.277771Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.739479Z","title":"Mind2web: Towards a generalist agent for the web","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.739479Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:742fcced577afa74ee7400d7e949a010a3cb38f8e65db82c238413d9c4a03365","observation_id":"627a74e2-98d7-42b2-929e-0df52d2e2100","resolution":{"observed_at":"2026-08-06T14:45:42.739479Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.21506","last_updated":"2025-07-03T15:47:40Z","snapshot_observed_at":"2026-08-07T20:30:36.688999Z","submitted_at":"2025-06-26T17:32:50Z","title":"Mind2Web 2: Evaluating Agentic Search with Agent-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.21506","snapshot_observed_at":"2026-08-06T14:45:42.742049Z","title":"Mind2web 2: Evaluating agentic search with agent-as-a-judge","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.742049Z"},"links":{"cited_paper":"/paper/2506.21506","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2cf50519c4e151757c3c56aa41b8585572936fc0fa9c993bf8c6cd5c4ce45070","observation_id":"7592be3c-8ba4-4e30-8d6d-1e4ef8e92ddf","resolution":{"observed_at":"2026-08-06T14:45:42.742049Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.04006","last_updated":"2020-11-08T15:53:56Z","snapshot_observed_at":"2026-07-06T10:12:38.348940Z","submitted_at":"2020-11-08T15:53:56Z","title":"Long Range Arena: A Benchmark for Efficient Transformers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.04006","snapshot_observed_at":"2026-08-06T14:45:42.745219Z","title":"Long range arena: A benchmark for efficient transformers","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.745219Z"},"links":{"cited_paper":"/paper/2011.04006","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:b110fe5c344a72fc3eec92b1f23313e78656120637b7804a916a798ac90cddcf","observation_id":"7bed5f91-00c3-4549-b38e-a7350617a961","resolution":{"observed_at":"2026-08-06T14:45:42.745219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.03358","last_updated":"2025-06-09T14:31:04Z","snapshot_observed_at":"2026-08-07T06:30:40.279371Z","submitted_at":"2025-02-05T16:53:45Z","title":"Minerva: A Programmable Memory Test Benchmark for Language Models","version":2},"cited_work":{"arxiv_id":"2502.03358","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.03358","snapshot_observed_at":"2026-08-06T14:45:42.844290Z","title":"Minerva: A Programmable Memory Test Benchmark for Language Models","venue":"cs.CL","work_id":"3f8ea708-e16b-48f0-9731-2e1b0891df17","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.748430Z"},"links":{"cited_paper":"/paper/2502.03358","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c5a70aa795a510fd71d7dd6903ae7b65a989ecf409af8677b0ab843df3e7ecf3","observation_id":"91e16ef8-01f0-4903-8d7c-ca63f47db4b5","resolution":{"observed_at":"2026-08-06T14:45:42.847863Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.261227Z","title":"L-eval: Instituting standardized evaluation for long context language models","venue":null,"work_id":"be5060db-c12c-4b48-8175-3dede473bde0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.751816Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:0943724ff646e6ad2e1369c50a7b419dc3c630a8d1bdc8676706eaa047ab1546","observation_id":"0b7abe6e-7add-4c05-9ba7-0a785830c544","resolution":{"observed_at":"2026-08-06T14:45:43.264284Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01437","last_updated":"2024-07-12T17:20:34Z","snapshot_observed_at":"2026-08-04T09:35:53.028512Z","submitted_at":"2024-07-01T16:32:16Z","title":"Needle in the Haystack for Memory Based Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01437","snapshot_observed_at":"2026-08-06T14:45:42.754635Z","title":"Needle in the haystack for memory based large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.754635Z"},"links":{"cited_paper":"/paper/2407.01437","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:52e9c9ef63398f912d48d4e7de6ac867591b7c531fad2f968798cf3776747ae6","observation_id":"af100c43-3815-4d90-a3df-597d7498c86d","resolution":{"observed_at":"2026-08-06T14:45:42.754635Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04681","last_updated":"2024-08-08T04:55:03Z","snapshot_observed_at":"2026-07-30T17:48:32.829011Z","submitted_at":"2024-08-08T04:55:03Z","title":"Conversational AI Powered by Large Language Models Amplifies False Memories in Witness Interviews","version":1},"cited_work":{"arxiv_id":"2408.04681","doi":null,"metadata_source":"pith","pith_arxiv_id":"2408.04681","snapshot_observed_at":"2026-08-06T14:45:42.820633Z","title":"Conversational AI Powered by Large Language Models Amplifies False Memories in Witness Interviews","venue":"cs.CL","work_id":"93769f36-8b0c-4227-a8e7-0d85d32a7df5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.757628Z"},"links":{"cited_paper":"/paper/2408.04681","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:24711aae5e2772fb76d756b78cb22ef582eab51a1115d1e9f21bc33269984563","observation_id":"a3c961f2-fb53-4177-96d8-a6b899f08052","resolution":{"observed_at":"2026-08-06T14:45:42.825972Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.252766Z","title":"Sentence-bert: Sentence embeddings using siamese bert-networks","venue":null,"work_id":"ec79078e-0857-4260-944f-df0816aa2b56","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.760596Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f5d5c26e98d5366b378f1d3b9ae53e6c66207512f1c527a5af9ac81bbb462a00","observation_id":"15d3363f-d81e-4c91-ab3b-69513f717179","resolution":{"observed_at":"2026-08-06T14:45:43.255832Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.243283Z","title":"Introduction to information retrieval , volume 39","venue":null,"work_id":"7ed06abb-43ee-447f-9ac5-7664f39ee98c","year":2008},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.763686Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ac3b9dfbae208f0189bc575d3c7d2ad389e49bb1e30783d52d8ad843d5e9bb63","observation_id":"e5677550-9e97-4b0e-9fda-11a30c4b5b32","resolution":{"observed_at":"2026-08-06T14:45:43.246950Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.234678Z","title":"The claude 3 model family: Opus, sonnet, haiku","venue":null,"work_id":"0bc329c8-d67c-4a4a-9911-2eb66e9732a7","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.766616Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ceec16a9b55a9165676086540328ba26db77504e6ef5168ff6efc683c7d23423","observation_id":"c7c6cf1c-fa05-4c7a-9208-b05183f4260b","resolution":{"observed_at":"2026-08-06T14:45:43.237288Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.226348Z","title":"Model card addendum: Claude 3.5 haiku and sonnet","venue":null,"work_id":"85d7a1e4-8284-499d-af7b-65a3d75b3e52","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.769395Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:8217888db9030607310a2fe28bd5ef5c519053a4ba0e6586440df73cf5b16d97","observation_id":"e6138f8e-1207-4806-a43c-7e73eb63aefa","resolution":{"observed_at":"2026-08-06T14:45:43.229332Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.05530","last_updated":"2024-12-16T17:39:39Z","snapshot_observed_at":"2026-07-06T17:41:42.995949Z","submitted_at":"2024-03-08T18:54:20Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.05530","snapshot_observed_at":"2026-08-06T14:45:42.772324Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.772324Z"},"links":{"cited_paper":"/paper/2403.05530","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:4551f795557a169a860ac4fb834fbc298b2b7695abf7afb9d87170d356e303ca","observation_id":"8918cd5d-be6f-4bd8-b8f9-282fccf6cdc2","resolution":{"observed_at":"2026-08-06T14:45:42.772324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.218646Z","title":"Introducing meta llama 3","venue":null,"work_id":"23b059f6-b0d9-4fdb-a5d4-aa9c5f99aa8e","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.775275Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:91c0d6fb65c071cde9737080219bb8c0c3a7ebdee1f0366cf349e9f81e7e6832","observation_id":"e4eee62f-382c-4feb-ad6a-30862ead2565","resolution":{"observed_at":"2026-08-06T14:45:43.221259Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.210394Z","title":"The serial position effect of free recall","venue":null,"work_id":"0ca13c23-e523-42be-8605-c2a74ce291f0","year":1962},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.778207Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:8fb48ba30ee8e7096fca673c89e5c3fad403ea2a6426dbd715a169a18f436dcb","observation_id":"de7165fe-97ad-4746-84c9-0e7cb86218a3","resolution":{"observed_at":"2026-08-06T14:45:43.213126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.201035Z","title":"luck-free","venue":null,"work_id":"6b354254-27b0-4920-aa2b-25962c330bd9","year":1966},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.780856Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:4c1bc94687a648c41cda47c9c4bbb348741094cad2dcad8926db07baf31003b2","observation_id":"1e0237f2-0b8c-4141-b590-1b0b2faa909c","resolution":{"observed_at":"2026-08-06T14:45:43.203784Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-06T14:36:11.929749Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models"},"reference_resolution":{"displayed":78,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":34,"verified_exact":4,"verified_fuzzy":40},"total_outbound_references":78},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 78 of 78 outbound references and 1 inbound Pith citation observation for arXiv:2507.18182."}