{"as_of":"2026-08-20T06:47:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:f3264096fb91c06255f2496962d21aad8fa0f5a971882a41a52478ebfc6ea44b","coverage":[{"denominator":66,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":66,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T14:58:40.410979Z","state":"measured"},{"denominator":66,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":66,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-20T06:33:59.587034+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02966/citation-record","integrity":"/paper/2608.02966/integrity","json":"/paper/2608.02966/citation-record.json","paper":"/paper/2608.02966"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.046108Z","title":"Darrell , title =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.046108Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:2ff65f115ff327bef948d1125aa962cc62d479240ce7c3f9e65fe4e66bd9519c","observation_id":"98a919b9-dec6-439a-b202-39dd39870100","resolution":{"observed_at":"2026-08-15T14:58:39.046108Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:44.183943Z","title":"Psychometrika , year =","venue":null,"work_id":"7a1c94f7-f905-4ed0-adf3-bd3db8f8f187","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.119070Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:d3894ba222ea10a62b3d3fae1bff588cb0f7fc6524c30f0c46348196984c18c4","observation_id":"49f83c5c-c3a7-4d2e-9b72-a61a452c5a80","resolution":{"observed_at":"2026-08-15T14:58:44.261192Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:44.075014Z","title":", title =","venue":null,"work_id":"d6e09ff7-30cc-4742-ba76-bba5bdac039b","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.207881Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:7cdc26044b557107b55058b405bf4db27802ae0fcdbe121c52412807620d8a91","observation_id":"e363868c-6342-4e44-b9f9-90da01d52558","resolution":{"observed_at":"2026-08-15T14:58:44.105716Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:44.055316Z","title":"Measurement: Interdisciplinary Research and Perspectives , year =","venue":null,"work_id":"75ab2c3e-32d2-4f23-acee-4532cd26819c","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.212976Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:5454c83a365b78e93806265cd37011fd6076280387fdf8350c0bd92a26e09293","observation_id":"1cfc82f6-00a7-46a8-9d4a-6880c92a96a5","resolution":{"observed_at":"2026-08-15T14:58:44.061099Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:44.036701Z","title":", title =","venue":null,"work_id":"01f42def-a7c0-4e71-a542-280b8ae3538e","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.217844Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:ed84fee4f3571f42b87f5a66d2b7055d0ad6d452680096ba6a83996d250ad929","observation_id":"42f866ec-59ac-42be-b44a-cba553a23282","resolution":{"observed_at":"2026-08-15T14:58:44.042634Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:44.019486Z","title":"Findings of the Association for Computational Linguistics: EMNLP 2023 , year =","venue":null,"work_id":"b0780711-af79-4e29-9858-cd68d1f46b70","year":2023},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.223126Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:51d8560e27cb0e871c80a48368e18dfbb462ac6fe39334f632b48413711b52a9","observation_id":"73ab8a78-50c6-439e-9029-d8baf7bd3019","resolution":{"observed_at":"2026-08-15T14:58:44.024830Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.227968Z","title":"Findings of the Association for Computational Linguistics: NAACL 2024 , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.227968Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:aea9511349c7bda6fba54a682062148f6334efb4fa811b6e9c927cc6a5ab5461","observation_id":"8f71685b-bdd6-4a45-85a3-68c9551f999a","resolution":{"observed_at":"2026-08-15T14:58:39.227968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.894292Z","title":"Journal of Educational Measurement , volume=","venue":null,"work_id":"39d70f8b-0058-4361-abda-5945c58f5a15","year":2003},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.232861Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:13b4d2686cdbce0a93fb1a156e98f5c1ded931d19ef2c2f4986b4d26c0a85a3a","observation_id":"4b1556e6-2d05-4252-bf2e-4dca6929eed9","resolution":{"observed_at":"2026-08-15T14:58:43.949220Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.06071","last_updated":"2025-02-08T10:48:33Z","snapshot_observed_at":"2026-08-16T13:36:06.120374Z","submitted_at":"2024-07-08T16:13:42Z","title":"From Loops to Oops: Fallback Behaviors of Language Models Under Uncertainty","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.06071","snapshot_observed_at":"2026-08-15T14:58:39.237142Z","title":"arXiv preprint arXiv:2407.06071 , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.237142Z"},"links":{"cited_paper":"/paper/2407.06071","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:b1c2a037f6a70a2274ca30a2e429c86376a407f2fbb9931061b533af651b9443","observation_id":"252a3a37-a1ed-4fc2-ada8-55cbaa1a1462","resolution":{"observed_at":"2026-08-15T14:58:39.237142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.819821Z","title":"and Linn, Robert L","venue":null,"work_id":"b35c6515-ce32-450e-a068-27d91cadbf85","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.243238Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:9cc2c224b54b30477744c7a89d4accc2ea8d947b0890f2756d86e74e3ae23bfd","observation_id":"bcc1d5fa-ab3e-4664-99a6-8b7354550dd0","resolution":{"observed_at":"2026-08-15T14:58:43.825304Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.800575Z","title":"Applied Psychological Measurement , volume=","venue":null,"work_id":"e9f356f8-b3d4-49ed-a6a7-1102cd14a0b5","year":2019},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.249743Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:f9c0f6fba3390e51ab7aa6bf220f6cd5c9fe382bcbacb050ff0fd9d8aa1abe26","observation_id":"466cfa14-f956-4623-8254-1127a8100af2","resolution":{"observed_at":"2026-08-15T14:58:43.806408Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.670994Z","title":"Marketing science , volume=","venue":null,"work_id":"990fba1e-e13f-442f-bc43-8797c4618fc6","year":2010},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.255201Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:40bc2f95e4f2a3215aa83ccaeeeb6e6de47afa8d2e251e8b2b314f9452366f57","observation_id":"09a27488-dcbf-43b2-b564-aad4156788fa","resolution":{"observed_at":"2026-08-15T14:58:43.724061Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.654264Z","title":"Transportation , volume=","venue":null,"work_id":"c3040bea-977a-4dac-92b3-c18cb40612cf","year":2012},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.259897Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:4e13783ab9b48ba0a89e0efa861ca9681d027b8a3dbdb137c43b1404a60a20f9","observation_id":"e80b7b7f-c848-4957-a7e2-d9148bee9788","resolution":{"observed_at":"2026-08-15T14:58:43.659772Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.265177Z","title":", title =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.265177Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:7cd6f1dc0e724f14aef5190103a2e695374ed262fb26f7634768b875601a5967","observation_id":"1887a2ca-bdfd-4c54-96e9-6fa6225cbb8a","resolution":{"observed_at":"2026-08-15T14:58:39.265177Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.533396Z","title":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics , year =","venue":null,"work_id":"cbab0db3-e227-4510-b037-e0902de89b76","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.269644Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:9df3bbdd722afba43be9184c28a89eb9a6e4496c2ea369ff6e22c7e16f50dd60","observation_id":"44b107b5-2ca9-427c-b3f0-5a49e1db1cad","resolution":{"observed_at":"2026-08-15T14:58:43.606732Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.274850Z","title":"Statistical Theories of Mental Test Scores , editor =","venue":null,"work_id":null,"year":1968},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.274850Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:34fc7d9413cb0cc6378f4019b98815ad2101f19529d11329059de69ca0181d12","observation_id":"275e80f0-bae8-4024-802b-a7d6a3c754d9","resolution":{"observed_at":"2026-08-15T14:58:39.274850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.497544Z","title":"Measurement: Interdisciplinary Research and Perspectives , volume=","venue":null,"work_id":"e5715ff6-7b1f-4b76-b875-482b772b617c","year":2009},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.279399Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:639f2ea9538e7e6dde4f27aa6f0e2bf588e9802feefa07adf4797d545a207ed1","observation_id":"eae2c5e3-1856-432e-9edc-2209300be1a2","resolution":{"observed_at":"2026-08-15T14:58:43.504572Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.479644Z","title":"On the Unidentifiability of the Fixed-Effects","venue":null,"work_id":"abe03b91-a432-44d4-b7b1-54df8ffaad28","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.283837Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:dbd088e6eea451b68f9792211952ebf9732c964425917375bffcc47a119f50af","observation_id":"ba14c083-016c-4271-872a-38a1c37b0263","resolution":{"observed_at":"2026-08-15T14:58:43.485264Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.353838Z","title":"and Lord, Frederic M","venue":null,"work_id":"6691b769-e086-40aa-9dcb-5264ef90e88e","year":1981},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.342230Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:1b1b82c8c42c3508e615dedd2c85eafb9cffb00f75ff2cb47c7b1df381745505","observation_id":"cc1e30ac-1a4f-4b26-a7c1-010f4fd28b8b","resolution":{"observed_at":"2026-08-15T14:58:43.433906Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.302897Z","title":"Philip and Skene, Allan M","venue":null,"work_id":"97592450-d20d-4d57-8ef7-f876c4856bde","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.442986Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:2ab03ee62a60cdddb7b32e2e453fdcaeb91b4815044731dc2d01cf382b61451a","observation_id":"e629055f-9993-48d0-8623-aef14d46fb21","resolution":{"observed_at":"2026-08-15T14:58:43.308092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.285695Z","title":"Advances in Neural Information Processing Systems 23 , year =","venue":null,"work_id":"13cf4dfa-ef05-48eb-8940-ab760a0f1893","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.491224Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:e1e65fdfe99f7782792c7f7feefbdbe4bf1b9f830f766492071ac10360c3b6c9","observation_id":"16490a9c-ed94-449d-871e-c36982697ff3","resolution":{"observed_at":"2026-08-15T14:58:43.290454Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.242389Z","title":"Advances in Neural Information Processing Systems 37, Datasets and Benchmarks Track , year =","venue":null,"work_id":"63c10ab0-c6d8-44b6-86e5-e9ef24c09b9b","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.496361Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:0f3342b0b1a217f0333e486a9293f11dd47fdf396742e7f8bced98c3afb90b56","observation_id":"31bdd3df-ac03-4c39-8953-b0efa414effd","resolution":{"observed_at":"2026-08-15T14:58:43.274140Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.149983Z","title":", title =","venue":null,"work_id":"7ed2bff8-45e2-4588-9240-42763bc1e1b4","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.501364Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:de6b7870b07e36f53bd22a96e6961a9e85d502114cb05f5b99421cb1754c3207","observation_id":"92b33539-a865-4026-8da0-581a48bf1772","resolution":{"observed_at":"2026-08-15T14:58:43.156418Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-08-14T19:36:07.505691Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-15T14:58:39.506630Z","title":"arXiv preprint arXiv:1803.05457 , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.506630Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:d1ab869e8431ff1237360669711bf1778933b3fa1ed204954fca408cfe46c73d","observation_id":"135fd128-a544-4b68-b6bf-2097d387b460","resolution":{"observed_at":"2026-08-15T14:58:39.506630Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:43.132913Z","title":"Findings of the Association for Computational Linguistics: NAACL 2024 , year =","venue":null,"work_id":"88e47ad4-5958-403c-a2c2-e9dcd49d64fe","year":2024},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.513083Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:965250448525c31a30e398049a9a2d9dbbaa0cc8bcceb55e65b6ba0d758a5575","observation_id":"bd1913b7-16ae-4c7d-8400-bdea67ff9f2e","resolution":{"observed_at":"2026-08-15T14:58:43.137872Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.957082Z","title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics , year =","venue":null,"work_id":"e9dc33ce-136b-4b85-943d-75a0f17f8251","year":2019},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.518553Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:f66b4a5988d19fff75d3d080620fe23ee5aa535f18bd0671c4a6cc6abb9afcc4","observation_id":"eb5a045f-c8d8-468f-96b1-7927be04d7cc","resolution":{"observed_at":"2026-08-15T14:58:43.045165Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.523190Z","title":"IEEE/ACM Transactions on Audio, Speech, and Language Processing , volume=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.523190Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:395e00c3fdf790abc4f12f145042c2fd1694dc6dc4abb90c84e1b65181293759","observation_id":"8449a83d-8cba-4f63-857a-b0b2aceab770","resolution":{"observed_at":"2026-08-15T14:58:39.523190Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.925722Z","title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing , year =","venue":null,"work_id":"94f5713d-b570-47d3-84fc-2c2d4e6633f5","year":2018},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.527766Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:c6debfb1217d562783f24e1416db75bba8b800c6e756ec282f8b6f5731339cc0","observation_id":"f74b1681-a0b0-4240-ba1f-1e5932b534c3","resolution":{"observed_at":"2026-08-15T14:58:42.931463Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.880395Z","title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics , year =","venue":null,"work_id":"1ea95c8c-8d2e-4c7a-97de-ff02257fb49a","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.532191Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:25f898ff7624f76152491a8a574615de8509c952a12822d76066f1fb362a95d1","observation_id":"d66e6103-db5e-4788-830c-b4514b182854","resolution":{"observed_at":"2026-08-15T14:58:42.913179Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.715951Z","title":"Applied Sciences , year =","venue":null,"work_id":"0cde08fa-2af6-419f-9460-9f300e81e825","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.537816Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:41588ff68e589f7b1947c3cf8444091a02e7a82d009fda0f0f4f7ba06d15533d","observation_id":"5730f55f-d330-4240-80b1-cb163c39e95c","resolution":{"observed_at":"2026-08-15T14:58:42.772270Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.698548Z","title":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing , year =","venue":null,"work_id":"2ed4ae17-300f-4292-8148-a5725d632e4e","year":2017},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.622501Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:8ea28322c21fa708be2db8576280d06a9058ecbb755bb4eaccc7690b5bba77be","observation_id":"743b2775-6ef4-40ad-a896-5649c658398e","resolution":{"observed_at":"2026-08-15T14:58:42.704715Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.695919Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.695919Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:b496dfb9a668eb07855a2c080985b8a08df5bf72a651d30b006c2b9bd0a175cf","observation_id":"3f39efbf-f6f1-4052-9402-79645ac5139f","resolution":{"observed_at":"2026-08-15T14:58:39.695919Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.670556Z","title":"Proceedings of the 34th AAAI Conference on Artificial Intelligence , year =","venue":null,"work_id":"83cd4687-228b-4577-880d-bbe7bc1b096c","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.753538Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:1b847e1c969fb718cd3f2728ba52045a70bec2413c1305a114cafbdd7552c955","observation_id":"823da1a1-170c-4777-a176-ccd8c8e3120c","resolution":{"observed_at":"2026-08-15T14:58:42.676083Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.635459Z","title":"International Conference on Learning Representations , year =","venue":null,"work_id":"bd3d338c-7ebb-452e-8301-fa50e03dc0f6","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.758422Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:60524573798ca7edc9ce02bb885328498b8def8d9682516b5ae455686d6a6a92","observation_id":"bf456cb8-86bf-45c3-bc28-2a0a9e346376","resolution":{"observed_at":"2026-08-15T14:58:42.659007Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.763766Z","title":"International Conference on Learning Representations , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.763766Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:889a19edc4400938d96388346e2ec83c66e1afa4c7b5cd64e2e22fae03aaf106","observation_id":"391415ce-7682-415f-9bf0-0c6e869888ab","resolution":{"observed_at":"2026-08-15T14:58:39.763766Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.456921Z","title":"Proceedings of the AAAI Conference on Artificial Intelligence , volume=","venue":null,"work_id":"1b809c6e-0b6d-4542-8fc2-e0635a040d79","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.768489Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:668cd21dfd5c23b79b28412dabb7b472767d477c88d7172ae04a85df56edafab","observation_id":"8b307df6-310e-49ec-a3a6-160c9715e38d","resolution":{"observed_at":"2026-08-15T14:58:42.537237Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:39.773095Z","title":"Probabilistic models for some intelligence and attainment tests","venue":null,"work_id":null,"year":1960},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.773095Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:56e68880340a45a87c0ffc307467d50ac975d4d37477fd1849f8673734dfec32","observation_id":"fd090313-a6fe-4f36-83e3-077afbdb9f52","resolution":{"observed_at":"2026-08-15T14:58:39.773095Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.419298Z","title":"1968 , publisher=","venue":null,"work_id":"123d1ba4-ce21-43a3-80e9-38832a082cf5","year":1968},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.778953Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:ce76416a00bc89a8e4d17924531f45eb80955b5fcc43b1467a7030c04db57abc","observation_id":"ac383f0e-602c-4944-a6ea-0f2e6cc0f6d1","resolution":{"observed_at":"2026-08-15T14:58:42.427621Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.389262Z","title":"Proceedings of the 12th International Conference on Educational Data Mining , year =","venue":null,"work_id":"97270990-e927-46c0-b96f-10bdadf93ed9","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.783899Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:10c3033cedf85c921e42960ef519a3e28862c5b0f64590d32e3e92d185d5a0bd","observation_id":"92de8232-a970-4f2f-9670-c82f5568a34e","resolution":{"observed_at":"2026-08-15T14:58:42.406286Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.226441Z","title":null,"venue":null,"work_id":"0e252afd-0112-43d2-9427-349b9d92c04d","year":2019},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.788953Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:5e2213ee0b4b701ff4a9d11fc79c30351ecefa4e751f059642580b9f1b7df027","observation_id":"891d7191-25f3-40f7-85c3-bbe2997d65e4","resolution":{"observed_at":"2026-08-15T14:58:42.274400Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.207602Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":"7fd4fac1-e9c5-4c05-90b5-3da4a79ebe32","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.794229Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:aaba261ca0987280efcab4ed30041345bf123b6c010d30295678733e8968b647","observation_id":"19025d95-827b-4ff1-a962-8d98ff1be4cd","resolution":{"observed_at":"2026-08-15T14:58:42.212821Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.108916Z","title":"2025 , eprint=","venue":null,"work_id":"ef97c644-0395-4278-a682-c65d05b80c55","year":2025},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.799355Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:338bcb23e2b3df18e12c808423f74cf163152b907564b8964773cfdf2c5fe0e7","observation_id":"7b740791-a193-4a6e-b833-e1e712d36b07","resolution":{"observed_at":"2026-08-15T14:58:42.145916Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2606.15643","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.310206Z","title":"arXiv preprint arXiv:2606.15643 , year=","venue":null,"work_id":"d6957b83-6691-4407-8466-57835d4664f5","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.803809Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:1354f7ed0f10e3736e09a2fc7ae4836b3f23d6ed5cf559306fc52ae08000c676","observation_id":"9be63788-8820-4dc1-bdb8-e98f3d805e94","resolution":{"observed_at":"2026-08-15T14:58:41.319337Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2511.04689","last_updated":"2026-07-13T22:56:21Z","snapshot_observed_at":"2026-08-18T18:20:27.842811Z","submitted_at":"2025-10-26T03:54:12Z","title":"Adaptive Testing for LLM Evaluation: A Psychometric Alternative to Static Benchmarks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2511.04689","snapshot_observed_at":"2026-08-15T14:58:39.837621Z","title":"arXiv preprint arXiv:2511.04689 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.837621Z"},"links":{"cited_paper":"/paper/2511.04689","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:af0c9dbe0d4f4868584cb7d1508b96786f0f2ae8cce72cef115215d14aa3e3c3","observation_id":"dcf39de4-b277-4ab4-af22-face4a94ea6a","resolution":{"observed_at":"2026-08-15T14:58:39.837621Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:42.092250Z","title":"Proceedings of the AAAI Conference on Artificial Intelligence , volume=","venue":null,"work_id":"8cb1708f-d96e-4f33-9283-f5c89fe80ecb","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:39.942519Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:0cb6c6ca4d08de9cbb2a96900ac7e0d7cd1465d3ee39e6eb8b9fbd56d4f72096","observation_id":"3914437c-a570-4816-99bd-f448b11d03fe","resolution":{"observed_at":"2026-08-15T14:58:42.097855Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.896603Z","title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing , pages=","venue":null,"work_id":"79d16fd9-a8ff-4720-b432-97150395432b","year":2025},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.025703Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:d4fca8aa8baced2064d0edd4dea6e1a1644c5be1675b2d85593e51d2e8f46fec","observation_id":"fc5cc222-e378-40ee-aa02-12ba49c7b4f9","resolution":{"observed_at":"2026-08-15T14:58:41.994121Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2207.05221","last_updated":"2022-11-21T16:38:35Z","snapshot_observed_at":"2026-08-14T05:42:29.067319Z","submitted_at":"2022-07-11T22:59:39Z","title":"Language Models (Mostly) Know What They Know","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.05221","snapshot_observed_at":"2026-08-15T14:58:40.046423Z","title":"arXiv preprint arXiv:2207.05221 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.046423Z"},"links":{"cited_paper":"/paper/2207.05221","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:f7195f02af65e1b1ee4601955418aed9e90c11ff4c7ef2b49c71d99a5f70ba09","observation_id":"48acfe07-7d24-4984-8462-7fa5334b4551","resolution":{"observed_at":"2026-08-15T14:58:40.046423Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.052098Z","title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing , pages=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.052098Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:5fab0fa16fffd60889c237124a311c1c1c6aa2a07de24417a1acdc1e495c716c","observation_id":"2a82a322-3441-4796-b949-4df879b38398","resolution":{"observed_at":"2026-08-15T14:58:40.052098Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.056761Z","title":"arXiv preprint arXiv:2509.10625 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.056761Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:da3e0d24047c9eb53d01a5d2f8f39eed01d4102ec1fd5b1f0ae90431ac91482d","observation_id":"8965e48b-1db1-41f6-b05f-85b037253ebd","resolution":{"observed_at":"2026-08-15T14:58:40.056761Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.858476Z","title":"Findings of the Association for Computational Linguistics: ACL 2025 , pages=","venue":null,"work_id":"b9362e9d-0afd-4e08-a240-21ec00cb88f5","year":2025},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.061858Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:a46c0333a51b9509564b1ff99c2837d67ec1ba9360e80e5e6a89817c6272980c","observation_id":"e320e6a1-cf39-4465-9788-2e469bfe716d","resolution":{"observed_at":"2026-08-15T14:58:41.865410Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2605.30504","last_updated":"2026-05-28T19:38:44Z","snapshot_observed_at":"2026-08-16T13:13:25.583251Z","submitted_at":"2026-05-28T19:38:44Z","title":"Auditing LLM Benchmarks with Item Response Theory","version":1},"cited_work":{"arxiv_id":"2605.30504","doi":null,"metadata_source":"pith","pith_arxiv_id":"2605.30504","snapshot_observed_at":"2026-08-15T14:58:40.858059Z","title":"Auditing LLM Benchmarks with Item Response Theory","venue":"cs.CL","work_id":"bf22ec97-0390-4ce3-a2c0-bb5fc055e2e4","year":2026},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.067379Z"},"links":{"cited_paper":"/paper/2605.30504","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:da92dcf522471c337109e1f57c3bbe4d281c174fa4600dc24f8ec341c02c3d57","observation_id":"9cfeaf8c-6499-47f7-a276-c9b42cfd7208","resolution":{"observed_at":"2026-08-15T14:58:40.949418Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.840321Z","title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing , pages=","venue":null,"work_id":"efcfa5e0-c22b-4f79-a3ca-94c51024fb1a","year":2025},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.072684Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:0cfc6d290cea4e485ede32c3a2b346db52c9f1e198c669dda7b5c25d70b01ddc","observation_id":"3009f9d1-f208-4363-8315-770cc243e049","resolution":{"observed_at":"2026-08-15T14:58:41.846317Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2606.17312","last_updated":"2026-06-15T21:39:47Z","snapshot_observed_at":"2026-08-09T16:35:02.364337Z","submitted_at":"2026-06-15T21:39:47Z","title":"Quantifying Consistency in LLM Logical Reasoning via Structural Uncertainty","version":1},"cited_work":{"arxiv_id":"2606.17312","doi":null,"metadata_source":"pith","pith_arxiv_id":"2606.17312","snapshot_observed_at":"2026-08-15T14:58:40.753819Z","title":"Quantifying Consistency in LLM Logical Reasoning via Structural Uncertainty","venue":"cs.AI","work_id":"3986d5e3-9fce-456b-82b5-d1bf022ee47c","year":2026},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.078037Z"},"links":{"cited_paper":"/paper/2606.17312","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:474155d6ad9cab4b1512603449407ab8abf3308f68b140416372fbf5c106bb26","observation_id":"c3269eb1-3b2f-4186-80ae-325b83e779cd","resolution":{"observed_at":"2026-08-15T14:58:40.759762Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.665423Z","title":"Proceedings of the AAAI Conference on Artificial Intelligence , volume=","venue":null,"work_id":"e97f0299-1379-4c48-bedb-01f18923cdc0","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.083482Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:8842ff41d94a768ddd549a0012837e00f6dd7b33585de7706255989b7b8b4981","observation_id":"86c54e22-70da-4285-8f37-224c27091867","resolution":{"observed_at":"2026-08-15T14:58:41.757465Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.644588Z","title":"International Conference on Learning Representations , volume=","venue":null,"work_id":"5a80a28d-8093-46b0-ae9e-6d7522d3a93a","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.170145Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:cdbacaf0b93324bb7f558746f11ea24e067e66ffa7c786d85fc7b867a86965f9","observation_id":"2b4193f8-9359-4f7d-a2d4-f298689cad4d","resolution":{"observed_at":"2026-08-15T14:58:41.652754Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.501322Z","title":null,"venue":null,"work_id":"a022d4b2-148a-4fad-85d5-c09f4660f60a","year":2025},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.220073Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:05293d61cdaf31617e29059678bc274ba8a477f7a7541ae374485eb00c3a7237","observation_id":"c9c76461-eb64-4550-8223-1c2edc8c8d8a","resolution":{"observed_at":"2026-08-15T14:58:41.573767Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-15T14:58:40.225030Z","title":"arXiv preprint arXiv:2403.04132 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.225030Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:80506befda0a33866ef946f2c2b7fabd38b3098429295b12d2ceb7ecca273d94","observation_id":"868a54fa-74fc-47cf-9642-0aa782cabb4d","resolution":{"observed_at":"2026-08-15T14:58:40.225030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2606.07422","last_updated":"2026-06-08T19:53:59Z","snapshot_observed_at":"2026-08-13T09:44:21.884297Z","submitted_at":"2026-06-05T16:16:59Z","title":"The Masked Advantage: Uncovering Local-Language Access to Cultural Knowledge in LLMs","version":2},"cited_work":{"arxiv_id":"2606.07422","doi":null,"metadata_source":"pith","pith_arxiv_id":"2606.07422","snapshot_observed_at":"2026-08-15T14:58:40.522204Z","title":"The Masked Advantage: Uncovering Local-Language Access to Cultural Knowledge in LLMs","venue":"cs.CL","work_id":"d532abde-6100-43c0-aabd-2c82573efa74","year":2026},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.229901Z"},"links":{"cited_paper":"/paper/2606.07422","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:99811d8a606fc657f3ce1c7b46d27c9b283ccca535f623d91fd0e373839a8b60","observation_id":"7b8055b6-ef0c-49f8-bf80-5f5a3343f606","resolution":{"observed_at":"2026-08-15T14:58:40.596362Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.235979Z","title":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing , pages =","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.235979Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:f35fb96caf0936fe3b72d47a7f0b2d7a45934c684c0c09821c06c6a47005416a","observation_id":"c31ed83a-6dba-456f-9667-c33999fe6cce","resolution":{"observed_at":"2026-08-15T14:58:40.235979Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.240467Z","title":"Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers) , pages =","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.240467Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:ea66e476249711153679ea85d60d6be79ff300fcc70644110a439de54b4a4933","observation_id":"fffebd34-82ae-4557-a7e8-05c872ba85c5","resolution":{"observed_at":"2026-08-15T14:58:40.240467Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.244955Z","title":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) , pages =","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.244955Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:8fb840f52c5c2447d4d1e110ac6f4e97da84cac03efcbd7c478a018490926440","observation_id":"5c26c898-a903-4f5e-ba48-c3b34c9fe424","resolution":{"observed_at":"2026-08-15T14:58:40.244955Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.252942Z","title":"Beyond Accuracy: Behavioral Testing of","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.252942Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:941167a19526cdd7d29c6dbaac383ec2f2f885bba154f3014cc1ec7e7fbeaf58","observation_id":"9d714cd8-41ec-4053-a260-51ae77bdf287","resolution":{"observed_at":"2026-08-15T14:58:40.252942Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:40.320183Z","title":"The Hitchhiker","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.320183Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:7d1ce2f5b8b59900f032b52647d8e24ce8522fbcc8d59e95bc4eb093043b43c7","observation_id":"fb4eb4cf-2d3e-442b-8b8b-53d72f4bdd99","resolution":{"observed_at":"2026-08-15T14:58:40.320183Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.481576Z","title":"The Spanish Journal of Psychology , volume=","venue":null,"work_id":"2d602e54-a661-4a9f-b990-19d30cd82302","year":2014},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.399395Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:e27c7542012be0fa7aea0a64b7dcd5f8e1f6d67a43c58a895343b1ff8852d8be","observation_id":"64d0d676-a841-4f1c-a9a0-97faae517a57","resolution":{"observed_at":"2026-08-15T14:58:41.487546Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14992","last_updated":"2024-05-26T22:27:23Z","snapshot_observed_at":"2026-08-17T02:59:38.954198Z","submitted_at":"2024-02-22T22:05:23Z","title":"tinyBenchmarks: evaluating LLMs with fewer examples","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14992","snapshot_observed_at":"2026-08-15T14:58:40.405231Z","title":"arXiv preprint arXiv:2402.14992 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.405231Z"},"links":{"cited_paper":"/paper/2402.14992","citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:8b0ec0fd7d3eb4ddabf096da55ec9c2f2b35e03307defe3362958acd6118b77f","observation_id":"50d02924-2b61-4d7c-bbc1-40e0c6886ba2","resolution":{"observed_at":"2026-08-15T14:58:40.405231Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T14:58:41.430055Z","title":"International Conference on Learning Representations , volume=","venue":null,"work_id":"e80593ba-b22a-4a17-8731-007ae84a7a2f","year":null},"citing_paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-15T14:58:40.410979Z"},"links":{"citing_paper":"/paper/2608.02966"},"observation_digest":"sha256:1b271b1b94b543591d0c22f08ea6de21ace372d71b53f845c001c65d1c36243b","observation_id":"7c5529b9-94e5-4a3a-bb49-163c8f2f68b5","resolution":{"observed_at":"2026-08-15T14:58:41.468974Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-20T06:33:59.587034+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2608.02966","last_updated":"2026-08-03T23:59:39Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-18T05:28:10.742527Z","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks"},"reference_resolution":{"displayed":66,"state_counts":{"malformed_identifier":0,"metadata_mismatch":3,"parse_uncertain":0,"unresolved":23,"verified_exact":1,"verified_fuzzy":39},"total_outbound_references":66},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-20T06:33:59.587034+00:00","source":"crossref"},{"observed_at":"2026-08-20T06:33:54.927442+00:00","source":"retraction_watch"}],"thesis":"As of 20 August 2026, this Paper Citation Record lists 66 of 66 outbound references and 0 inbound Pith citation observations for arXiv:2608.02966."}