{"as_of":"2026-08-18T07:50:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d388a75398d347337cd473df5d6b72bd8307d6b342d8add9ee1088e98e6c7f12","coverage":[{"denominator":70,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":70,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T17:07:19.897165Z","state":"measured"},{"denominator":71,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":71,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-18T06:34:40.430872+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-10T04:09:46.616019Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-11T12:06:04.092759Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"cited_work":{"arxiv_id":"2508.17580","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.17580","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv preprint arXiv:2508.17580 , year=","venue":null,"work_id":"036d55ea-22dd-43ce-a074-2fcb1cd8ffc6","year":null},"citing_paper":{"arxiv_id":"2604.18584","last_updated":"2026-07-29T01:04:08Z","snapshot_observed_at":"2026-08-14T09:52:37.857940Z","submitted_at":"2026-04-20T17:59:49Z","title":"MathNet: a Global Multimodal Benchmark for Mathematical Reasoning and Retrieval","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-05-10T04:09:46.616019Z"},"links":{"cited_paper":"/paper/2508.17580","citing_paper":"/paper/2604.18584"},"observation_digest":"sha256:02ae2f945655042b00737af3e1eb40cef5612e52ab593320c1bdcfe4195d41dc","observation_id":"51e8effd-fb12-4824-8bf6-19061be8cfac","resolution":{"observed_at":"2026-05-11T12:06:04.097499Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2508.17580/citation-record","integrity":"/paper/2508.17580/integrity","json":"/paper/2508.17580/citation-record.json","paper":"/paper/2508.17580"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.840437Z","title":"Piqa: Reasoning about physical commonsense in natural language","venue":null,"work_id":"bcab0fae-8c27-44ed-aaaf-8c04ce587e4c","year":2020},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.612229Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:dc2fbc36681eac25b23de4ae30a801dea62d45fc33f02ab641eab5c5be30807a","observation_id":"1f01bbf6-b12f-46b1-b9ae-2e3b1a5845af","resolution":{"observed_at":"2026-08-15T17:07:20.844600Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T17:07:19.617165Z","title":"Evaluating large language models trained on code","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.617165Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:2f26fb718c1acd19c8349269e48778f20b8116dadd73c3c0d3b778d9e2030295","observation_id":"5a54b432-f1ee-46b1-bbef-b1f27ff77a20","resolution":{"observed_at":"2026-08-15T17:07:19.617165Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.828070Z","title":"Chatbot arena: An open platform for evaluating llms by human preference","venue":null,"work_id":"ef2febb2-f2c4-4aac-81a5-978b8fe46f46","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.621341Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:c2d0753ad07fd44c6ee744693f2f392046d53785ea6fc5d7ef90b47394ff18b1","observation_id":"7c1e60d2-ce2f-4c88-b390-652795b68d92","resolution":{"observed_at":"2026-08-15T17:07:20.832642Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.04604","last_updated":"2025-01-08T05:24:50Z","snapshot_observed_at":"2026-08-16T13:00:34.704005Z","submitted_at":"2024-12-05T20:40:28Z","title":"ARC Prize 2024: Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.04604","snapshot_observed_at":"2026-08-15T17:07:19.625583Z","title":"Arc prize 2024: Technical report","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.625583Z"},"links":{"cited_paper":"/paper/2412.04604","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:a0625f078f4aa36820d23e0e466f9dd6a82d539b4f623bedad96d1eb24facb4c","observation_id":"e4e6170e-de5a-4d34-b437-6df8349d1a78","resolution":{"observed_at":"2026-08-15T17:07:19.625583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-15T17:07:19.629700Z","title":"Training verifiers to solve math word problems","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.629700Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:7b94637b51e17dfc7fd73b2b1bd66124e9bbc02be38e1dc13769482713dc910e","observation_id":"2176fa22-ebda-4452-93cd-285dcdc6d287","resolution":{"observed_at":"2026-08-15T17:07:19.629700Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11495","last_updated":"2023-09-25T15:25:49Z","snapshot_observed_at":"2026-08-12T01:35:36.884518Z","submitted_at":"2023-09-20T17:50:55Z","title":"Chain-of-Verification Reduces Hallucination in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11495","snapshot_observed_at":"2026-08-15T17:07:19.634145Z","title":"Chain-of-verification reduces hallucination in large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.634145Z"},"links":{"cited_paper":"/paper/2309.11495","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:8ad5a2397b6c4d3e202d1f536e37ab7fe35ce700e19beb7179a772b2099d4aa7","observation_id":"5b463a90-bfec-45d1-b90e-d542c6e823f5","resolution":{"observed_at":"2026-08-15T17:07:19.634145Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14387","last_updated":"2024-01-08T04:46:56Z","snapshot_observed_at":"2026-08-16T17:34:14.000776Z","submitted_at":"2023-05-22T17:55:50Z","title":"AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14387","snapshot_observed_at":"2026-08-15T17:07:19.638212Z","title":"Alpacafarm: A simulation framework for methods that learn from human feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.638212Z"},"links":{"cited_paper":"/paper/2305.14387","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:43a7b7fc91d00dfb9d0ede7517b4fccc13ceb184947790390dcc97f10533341a","observation_id":"b0069902-9909-4c98-8a27-57bfafb52b46","resolution":{"observed_at":"2026-08-15T17:07:19.638212Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04127","last_updated":"2025-01-10T14:31:21Z","snapshot_observed_at":"2026-08-16T13:45:30.496997Z","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04127","snapshot_observed_at":"2026-08-15T17:07:19.642257Z","title":"Are we done with mmlu? arXiv preprint arXiv:2406.04127 , 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.642257Z"},"links":{"cited_paper":"/paper/2406.04127","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:878583315be9c242dd6d52bcb49f37ea9763c97d56f08d50b20577169ed93f67","observation_id":"7f96f91e-c4d8-41ae-a264-112d60ea3305","resolution":{"observed_at":"2026-08-15T17:07:19.642257Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.646336Z","title":"Frontiermath: A benchmark for evaluating advanced mathematical reasoning in ai, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.646336Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:481c29ce1d17ac73a72c36170b1d3ecc83388a7ad27785d66da9fb23e8edf604","observation_id":"f1ab611a-d399-4435-a64c-f27784877ef4","resolution":{"observed_at":"2026-08-15T17:07:19.646336Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.04313","last_updated":"2025-06-12T14:43:36Z","snapshot_observed_at":"2026-08-18T06:47:00.768909Z","submitted_at":"2025-02-06T18:56:01Z","title":"Great Models Think Alike and this Undermines AI Oversight","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.04313","snapshot_observed_at":"2026-08-15T17:07:19.653111Z","title":"Great models think alike and this undermines ai oversight","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.653111Z"},"links":{"cited_paper":"/paper/2502.04313","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:44bf5fc92c8187edf0577e79d9601698228311e97e1c93c1dee51b9197aeeb94","observation_id":"593bde5f-a009-44e0-89ef-0bb417085f93","resolution":{"observed_at":"2026-08-15T17:07:19.653111Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-14T15:19:05.440904Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-15T17:07:19.657157Z","title":"Measuring coding challenge competence with apps","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.657157Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:d5f1a5f0326a17f6533a9ce14f176ee5d9fd5e3081aa64bbdf56f550731b975f","observation_id":"cd69e84c-1409-4aa7-8517-78fe5e4d119c","resolution":{"observed_at":"2026-08-15T17:07:19.657157Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-15T17:07:19.661079Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.661079Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:b1130d2f86eb9d04f2e42bdc0ccef109b0d7b91c79bc44bfb652acdb51077164","observation_id":"a09cdff2-45c3-4aa9-bf4b-68c6d2d6426a","resolution":{"observed_at":"2026-08-15T17:07:19.661079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.03874","last_updated":"2021-11-08T21:30:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-03-05T18:59:39Z","title":"Measuring Mathematical Problem Solving With the MATH Dataset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.03874","snapshot_observed_at":"2026-08-15T17:07:19.664976Z","title":"Measuring mathematical problem solving with the math dataset","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.664976Z"},"links":{"cited_paper":"/paper/2103.03874","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:ad61fbaf89eba57525df3a4be3895e90cdf9fff817ebb11c3faed73dd603aa5d","observation_id":"fcbcf013-b427-4ecb-95de-df825ee3cb95","resolution":{"observed_at":"2026-08-15T17:07:19.664976Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06453","last_updated":"2025-02-12T23:16:27Z","snapshot_observed_at":"2026-08-16T00:12:44.179679Z","submitted_at":"2025-02-10T13:31:46Z","title":"MATH-Perturb: Benchmarking LLMs' Math Reasoning Abilities against Hard Perturbations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06453","snapshot_observed_at":"2026-08-15T17:07:19.669042Z","title":"Math-perturb: Benchmarking llms' math reasoning abilities against hard perturbations","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.669042Z"},"links":{"cited_paper":"/paper/2502.06453","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:09bfd48421af79454a64c3d6b38ddc7cd214556274b26c552b6a84f0a84d7f41","observation_id":"79c6791a-03b7-4619-aa14-76667eb6461e","resolution":{"observed_at":"2026-08-15T17:07:19.669042Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.07493","last_updated":"2025-01-13T17:12:38Z","snapshot_observed_at":"2026-08-15T14:44:39.390234Z","submitted_at":"2025-01-13T17:12:38Z","title":"Exploring and Mitigating Adversarial Manipulation of Voting-Based Leaderboards","version":1},"cited_work":{"arxiv_id":"2501.07493","doi":null,"metadata_source":"pith","pith_arxiv_id":"2501.07493","snapshot_observed_at":"2026-08-15T17:07:20.359735Z","title":"Exploring and Mitigating Adversarial Manipulation of Voting-Based Leaderboards","venue":"cs.LG","work_id":"68edfe29-6ebb-4853-bd29-77cebf622dbc","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.673009Z"},"links":{"cited_paper":"/paper/2501.07493","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:c0c962d4709ead3d50f316dc872ebd2b9c3f5ada07049c939537040549bd6575","observation_id":"7cfd3e5b-372f-4122-b6cc-97d91e78eacc","resolution":{"observed_at":"2026-08-15T17:07:20.365810Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21276","last_updated":"2024-10-25T17:43:01Z","snapshot_observed_at":"2026-08-15T14:02:47.366139Z","submitted_at":"2024-10-25T17:43:01Z","title":"GPT-4o System Card","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21276","snapshot_observed_at":"2026-08-15T17:07:19.677086Z","title":"Gpt-4o system card","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.677086Z"},"links":{"cited_paper":"/paper/2410.21276","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:27606822305a369a774656aa714c2a08d5368dce148423219f41b192807bdd33","observation_id":"9effe6b5-3a3f-4cf8-b0bf-7c9cf841f69b","resolution":{"observed_at":"2026-08-15T17:07:19.677086Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-16T07:05:57.323612Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-15T17:07:19.681246Z","title":"Livecodebench: Holistic and contamination free evaluation of large language models for code","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.681246Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:0df1c245d603676f37c0778b429307fa635d05acdb7036b49c1fbbc9580f10d8","observation_id":"1d031044-a077-4dd3-af42-2e333a3d4d62","resolution":{"observed_at":"2026-08-15T17:07:19.681246Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.10511","last_updated":"2023-07-02T07:21:59Z","snapshot_observed_at":"2026-07-06T14:33:08.041820Z","submitted_at":"2022-12-20T18:30:15Z","title":"When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.10511","snapshot_observed_at":"2026-08-15T17:07:19.685883Z","title":"Lm vs lm: Detecting factual errors in language models using language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.685883Z"},"links":{"cited_paper":"/paper/2212.10511","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:06d0c5f1f288ebb286f1a7d9577f6a3bef1ea7d1c94372b22465bea68fe44231","observation_id":"971742a0-244e-4bb1-ac3d-08e9653d5389","resolution":{"observed_at":"2026-08-15T17:07:19.685883Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.690864Z","title":"Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik Narasimhan","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.690864Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:8b21722911e10f78c5d9833a94221671867da2e2b4fc520e43e3c2439ab140d2","observation_id":"aa30218c-fcfd-4868-90e8-1ec7d03d2667","resolution":{"observed_at":"2026-08-15T17:07:19.690864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.695712Z","title":"Verdict: A library for scaling judge-time compute","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.695712Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:46b8f8afad3a8e9ceef33933ae585ba67944222414cfeda22c5469c4bb793261","observation_id":"e32cb678-272d-4129-852e-ff4739ee86ff","resolution":{"observed_at":"2026-08-15T17:07:19.695712Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.801236Z","title":"Llm-as-an-interviewer: Beyond static testing through dynamic llm evaluation, 2025","venue":null,"work_id":"3bc4cf53-9bea-4b99-979d-bfed9d62ed87","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.700099Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:b7c52845ba54ef83f6ce016f9e041d550c2e1a48a735875af066415f345f3604","observation_id":"190333ed-45ec-463e-9577-a67c0a8fdb35","resolution":{"observed_at":"2026-08-15T17:07:20.805990Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.703854Z","title":"Prometheus: Inducing fine-grained evaluation capability in language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.703854Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:4dcd093bf5a48038306c336c2350d166bd1d4fbdd4c43c132018a522139d1b09","observation_id":"158af6cb-3842-49c6-a647-0dd64f7d1c0d","resolution":{"observed_at":"2026-08-15T17:07:19.703854Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.781998Z","title":"Prometheus 2: An open source language model specialized in evaluating other language models, 2024","venue":null,"work_id":"bb03383f-39d6-4f44-810c-1f9162337ae4","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.707492Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:9993787d4f804735a05b7955a16eef1ffd46f0873e7b101a5e1073e1fe9a8875","observation_id":"519f40fe-6840-4a3d-98a5-c3c18e81dd31","resolution":{"observed_at":"2026-08-15T17:07:20.785977Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.769208Z","title":"Scaling evaluation-time compute with reasoning models as process evaluators, 2025","venue":null,"work_id":"762da230-f5f0-4496-934e-aa414f1fbcd3","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.711145Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:9d8201a915295d5d52409842b838919816f1157523077d8146c5b0a25d696f8c","observation_id":"b93b5b2a-5278-4db2-b564-4f15cb79e6ef","resolution":{"observed_at":"2026-08-15T17:07:20.773770Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.757575Z","title":"Toutanova, Llion Jones, Ming-Wei Chang, Andrew Dai, Jakob Uszkoreit, Quoc Le, and Slav Petrov","venue":null,"work_id":"5e5a7478-9226-405f-9b39-8d7ae0dfcce6","year":2019},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.714954Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:2ef8f0f8236a0f16e20c523a28c30454734faa3f6525960db8b095d28b6c3a10","observation_id":"b77f1e55-ca6b-4450-ac93-948aef2b3bbc","resolution":{"observed_at":"2026-08-15T17:07:20.761723Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.744949Z","title":"The measurement of observer agreement for categorical data","venue":null,"work_id":"031e01df-af06-4217-b306-276a94d5a5dd","year":1977},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.718615Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:78c419695a9838031bc20846e69d68fa27b7c82b207a9aab76bfe1d7efd7b43d","observation_id":"b75cb40a-367c-4ecf-9060-cdf0a7d783da","resolution":{"observed_at":"2026-08-15T17:07:20.749131Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.10928","last_updated":"2024-04-14T04:29:51Z","snapshot_observed_at":"2026-08-16T15:14:38.528933Z","submitted_at":"2023-07-20T14:56:35Z","title":"FLASK: Fine-grained Language Model Evaluation based on Alignment Skill Sets","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.10928","snapshot_observed_at":"2026-08-15T17:07:19.722330Z","title":"Flask: Fine-grained language model evaluation based on alignment skill sets","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.722330Z"},"links":{"cited_paper":"/paper/2307.10928","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:afeb1b4eb60600f56a49921ab730eff62e233e9dcb13eac2e6508cc9449d52aa","observation_id":"ddd119d8-450b-4eb6-b495-24bfb2c8b879","resolution":{"observed_at":"2026-08-15T17:07:19.722330Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T17:07:19.726851Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.726851Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:2cc27bac0d5def2b8451e1498f13c6c9bb8d385cc065501dd64c705ab78a6fa6","observation_id":"c5edad0b-6467-463f-8ec4-68fefa0054bc","resolution":{"observed_at":"2026-08-15T17:07:19.726851Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.20050","last_updated":"2023-05-31T17:24:00Z","snapshot_observed_at":"2026-08-17T09:42:34.746112Z","submitted_at":"2023-05-31T17:24:00Z","title":"Let's Verify Step by Step","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.20050","snapshot_observed_at":"2026-08-15T17:07:19.730930Z","title":"Let's verify step by step","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.730930Z"},"links":{"cited_paper":"/paper/2305.20050","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:21674b9eca387e267267e02c506412a51ff3d1bb1829e4fccec15a7f5e03163c","observation_id":"374897c1-dee4-4c24-be9c-51e809777187","resolution":{"observed_at":"2026-08-15T17:07:19.730930Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04770","last_updated":"2024-10-05T22:39:51Z","snapshot_observed_at":"2026-08-16T13:45:13.487794Z","submitted_at":"2024-06-07T09:15:44Z","title":"WildBench: Benchmarking LLMs with Challenging Tasks from Real Users in the Wild","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04770","snapshot_observed_at":"2026-08-15T17:07:19.734688Z","title":"Wildbench: Benchmarking llms with challenging tasks from real users in the wild","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.734688Z"},"links":{"cited_paper":"/paper/2406.04770","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:6a83f0ba743d05e12139de2362feae188c1b6ebc381eb28c1d2cdc820ea8be3b","observation_id":"8cfc8a7c-e9e9-4a2e-a6f8-b88c3c7ca7da","resolution":{"observed_at":"2026-08-15T17:07:19.734688Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.09848","last_updated":"2023-10-23T19:11:38Z","snapshot_observed_at":"2026-08-16T15:39:11.826409Z","submitted_at":"2023-04-19T17:56:12Z","title":"Evaluating Verifiability in Generative Search Engines","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.09848","snapshot_observed_at":"2026-08-15T17:07:19.738927Z","title":"Evaluating verifiability in generative search engines","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.738927Z"},"links":{"cited_paper":"/paper/2304.09848","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:0c379279d2b47b97f03450f0bd26f9f05bc7bbd792b7d6a2465f079a3622c1eb","observation_id":"3f5e8e7a-8fe5-4d40-98c1-2638cc7c8139","resolution":{"observed_at":"2026-08-15T17:07:19.738927Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.732303Z","title":"The lean 4 theorem prover and programming language","venue":null,"work_id":"e835c275-f384-494d-9d29-378483bc40d1","year":2021},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.742881Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:c82fe8d42e5ad237ff310f1ec39e4293c930b2cde442396f05d3fd874b7966c5","observation_id":"5ef7b271-c72c-4a61-8ea2-4d5fe30ccce6","resolution":{"observed_at":"2026-08-15T17:07:20.736175Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.714739Z","title":"2024 aime i","venue":null,"work_id":"88813d07-a5c2-4961-861d-bb674a8fa740","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.747278Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:833b867d0fd9ae6c96f738f7676a1c79a8e90b06eb6764c6ef8db497b178e1b1","observation_id":"b9ea21ba-ee35-4da9-a8cb-eb220e3c4895","resolution":{"observed_at":"2026-08-15T17:07:20.723466Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.700598Z","title":"List of open problems in sublinear algorithms","venue":null,"work_id":"7643f28a-a9cd-47a1-966a-7de47608bb94","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.751098Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:073d5aaa8d7bcdd4f1ffd0685081ed1046befb676b569ae60dfb37d530662612","observation_id":"c95d7ef9-422e-45a1-99ef-461c49e60bd0","resolution":{"observed_at":"2026-08-15T17:07:20.705112Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.682230Z","title":"Introducing deep research","venue":null,"work_id":"a34db4dc-7689-4ca9-be86-4f3037342978","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.754764Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:96f991a529acc890cdb039027f5a50e8170c6287f98fc3bf7768e51f355aebfe","observation_id":"4a628fec-1c3f-4286-8a62-7c19ff390b07","resolution":{"observed_at":"2026-08-15T17:07:20.687109Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.666293Z","title":"Introducing OpenAI o1","venue":null,"work_id":"fafb346d-8529-4801-8478-8dcea25bf1da","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.758160Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:e5690c3f796586d57bed2f55dd9083cbf3ab5ebad3d24b4235b50444fda3d7af","observation_id":"6c25daa6-4998-412a-aae5-bf9e5fbdf080","resolution":{"observed_at":"2026-08-15T17:07:20.671042Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.654471Z","title":"Introducing OpenAI o3 and o4‑mini","venue":null,"work_id":"531e9534-e13b-48b2-a0d8-4d9febdfba33","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.761596Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:f4426ac63d5b31635402e43793be559ea4c67f1e4a87677f50998074c43ff47d","observation_id":"e3eceb3f-2fb3-406b-a739-7e4813bb1314","resolution":{"observed_at":"2026-08-15T17:07:20.658233Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.642856Z","title":"Llm evaluators recognize and favor their own generations","venue":null,"work_id":"d6de9bc9-a419-41be-a82e-961d1ceea6f3","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.765577Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:171268e85dc418304f82890fa137c580753a712556afefe84e7e3f4d1f685807","observation_id":"deceb3d9-7c72-4a20-9e7f-d229cbc500aa","resolution":{"observed_at":"2026-08-15T17:07:20.646999Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.630922Z","title":"For better or worse, benchmarks shape a field","venue":null,"work_id":"4f01d4ab-bf8e-4e3b-b4c6-f7798fe82bba","year":2012},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.769878Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:b78de2c0a4c187e8f4f9f347c9ed4b44fbd0c3eece8ae68804031241e229ee1c","observation_id":"56d7c659-603b-4be9-bd3b-05300528dfcf","resolution":{"observed_at":"2026-08-15T17:07:20.635111Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.02252","last_updated":"2021-05-27T15:20:59Z","snapshot_observed_at":"2026-08-15T21:54:51.990010Z","submitted_at":"2020-09-04T15:32:19Z","title":"KILT: a Benchmark for Knowledge Intensive Language Tasks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.02252","snapshot_observed_at":"2026-08-15T17:07:19.774679Z","title":"Kilt: a benchmark for knowledge intensive language tasks","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.774679Z"},"links":{"cited_paper":"/paper/2009.02252","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:85dd8d92ea0c234cd67d2df3f429cbc9e1e15f6d460ef2053666504747573ef9","observation_id":"5c2371bb-a85c-4f62-af4c-210971e40705","resolution":{"observed_at":"2026-08-15T17:07:19.774679Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.618166Z","title":"Humanity's last exam, 2025","venue":null,"work_id":"d8346d64-fa3d-42a5-9f4c-ec49631e8ebd","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.779121Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:45e6dad5b71c845fb1b9c6a04f5708e4236928b624954188e2518ef7908aad39","observation_id":"2197aa18-381d-4f4b-8320-10369cff45bd","resolution":{"observed_at":"2026-08-15T17:07:20.622441Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1606.05250","last_updated":"2016-10-11T02:42:36Z","snapshot_observed_at":"2026-08-17T14:32:22.468812Z","submitted_at":"2016-06-16T16:36:00Z","title":"SQuAD: 100,000+ Questions for Machine Comprehension of Text","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1606.05250","snapshot_observed_at":"2026-08-15T17:07:19.783306Z","title":"Squad: 100,000+ questions for machine comprehension of text","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.783306Z"},"links":{"cited_paper":"/paper/1606.05250","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:06444f5cb2f3877b6b891174c0da60907dea040faf878904d967f21989b567fb","observation_id":"fd37a82f-a72d-47b4-bf2a-36bb7c1ad53f","resolution":{"observed_at":"2026-08-15T17:07:19.783306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.605982Z","title":"Gpqa: A graduate-level google-proof q&a benchmark","venue":null,"work_id":"1ce7671e-541b-4320-b992-9e67058ea4fe","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.787188Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:60006618b11951e8abf728773cfb79fd4324687dbf341a8fd82a27ec716bb4c2","observation_id":"6218fdfb-c2e4-45b0-9ea3-dc64640d69ae","resolution":{"observed_at":"2026-08-15T17:07:20.609957Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.10573","last_updated":"2025-06-26T16:38:11Z","snapshot_observed_at":"2026-08-15T21:41:44.521608Z","submitted_at":"2025-05-13T20:36:22Z","title":"Measurement to Meaning: A Validity-Centered Framework for AI Evaluation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.10573","snapshot_observed_at":"2026-08-15T17:07:19.792055Z","title":"Measurement to meaning: A validity-centered framework for ai evaluation","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.792055Z"},"links":{"cited_paper":"/paper/2505.10573","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:c2d7f238d3d5e26ba7e0d4ee196164b4b979226be3be9d71e942a7bfd8ca19a0","observation_id":"326f22e5-6d1b-4715-85f1-3cc44efa40da","resolution":{"observed_at":"2026-08-15T17:07:19.792055Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.20879","last_updated":"2025-05-12T16:33:58Z","snapshot_observed_at":"2026-08-17T00:43:37.683523Z","submitted_at":"2025-04-29T15:48:49Z","title":"The Leaderboard Illusion","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.20879","snapshot_observed_at":"2026-08-15T17:07:19.796329Z","title":"The leaderboard illusion","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.796329Z"},"links":{"cited_paper":"/paper/2504.20879","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:4e9150760429b3abec39027fba4edda48d975866ba4fa270e91338a31652764c","observation_id":"fb828bae-2d44-495f-8143-88060708b8ea","resolution":{"observed_at":"2026-08-15T17:07:19.796329Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.594633Z","title":"Stack Exchange","venue":null,"work_id":"2d0deb23-c3f9-4b77-88fb-71c3fcc0727b","year":null},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.800390Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:77f581b234b7a5dce0fd97721cc1ec631bfc82bf8cf2b8ae7adf392fa963c260","observation_id":"1608eedb-7ee9-4894-8458-1124658ba64f","resolution":{"observed_at":"2026-08-15T17:07:20.598457Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.582035Z","title":"Terminal‑Bench : A benchmark for ai agents in terminal environments","venue":null,"work_id":"247ea36c-1f52-4a13-b5a7-31f355811f0a","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.804671Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:800aa9ddd3dcc89560b7167fde9623fbfc67a9c5670807eaebe0d224cd72d790","observation_id":"0410b2a1-1b7c-4dad-9a8f-d2539af0772e","resolution":{"observed_at":"2026-08-15T17:07:20.586555Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.23009","last_updated":"2026-05-11T11:47:15Z","snapshot_observed_at":"2026-08-14T11:42:36.697068Z","submitted_at":"2025-07-30T18:14:35Z","title":"Position: Stop Evaluating AI with Human Tests, Develop Principled, AI-specific Tests instead","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.23009","snapshot_observed_at":"2026-08-15T17:07:19.808716Z","title":"Stop evaluating ai with human tests, develop principled, ai-specific tests instead","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.808716Z"},"links":{"cited_paper":"/paper/2507.23009","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:45fde431d972c9680b286d60ba3f69236526681128404ac041a7fb517d3df3ce","observation_id":"0f163747-562a-4de4-97a2-59c4637bf9e5","resolution":{"observed_at":"2026-08-15T17:07:19.808716Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.564460Z","title":"Supergpqa: Scaling llm evaluation across 285 graduate disciplines, 2025","venue":null,"work_id":"205d0d3e-413a-4ec2-84a7-e26cfe6fadf8","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.812701Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:7491fc6c11f15937caf0a91d6c97876d51543f6710fd17f07f2ddbe76f698819","observation_id":"61312f11-fa5f-47f3-b869-c30b2ab19004","resolution":{"observed_at":"2026-08-15T17:07:20.572183Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1905.00537","last_updated":"2020-02-13T00:28:00Z","snapshot_observed_at":"2026-08-17T22:08:45.245439Z","submitted_at":"2019-05-02T00:41:50Z","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.00537","snapshot_observed_at":"2026-08-15T17:07:19.816806Z","title":"Superglue: A stickier benchmark for general-purpose language understanding systems","venue":null,"work_id":null,"year":1905},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.816806Z"},"links":{"cited_paper":"/paper/1905.00537","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:79814df4d7a4f9cb0ca6abd75d3648da4f735027d8825f0e90199bfaf02fc35c","observation_id":"5716fd16-daec-44b3-8206-cc4a9af77846","resolution":{"observed_at":"2026-08-15T17:07:19.816806Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1804.07461","last_updated":"2019-02-22T23:53:34Z","snapshot_observed_at":"2026-08-16T09:50:11.319379Z","submitted_at":"2018-04-20T06:35:04Z","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1804.07461","snapshot_observed_at":"2026-08-15T17:07:19.820943Z","title":"Glue: A multi-task benchmark and analysis platform for natural language understanding","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.820943Z"},"links":{"cited_paper":"/paper/1804.07461","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:0aa6826e8c272c0d330a261ded74ee0844475a4859323694445910c4fdcd38f3","observation_id":"d5a02e5a-b954-4fb5-99a2-6e485959b2f8","resolution":{"observed_at":"2026-08-15T17:07:19.820943Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.824905Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.824905Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:3019fbb2c1156c92e8e7f2fb06ff2a0e39983dc3ce8549dfe6e9bea2b89a617b","observation_id":"af631803-152b-46b6-9303-cd34c843b03e","resolution":{"observed_at":"2026-08-15T17:07:19.824905Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05087","last_updated":"2024-05-24T06:37:31Z","snapshot_observed_at":"2026-08-16T15:25:32.155059Z","submitted_at":"2023-06-08T10:41:56Z","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05087","snapshot_observed_at":"2026-08-15T17:07:19.828751Z","title":"Pandalm: An automatic evaluation benchmark for llm instruction tuning optimization","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.828751Z"},"links":{"cited_paper":"/paper/2306.05087","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:579bab474381b4ed8ad92d5ad7c9909044ebc09c51641db3a6f0b5a942365399","observation_id":"24f5b672-60a7-49ca-80f8-3ab96fef1669","resolution":{"observed_at":"2026-08-15T17:07:19.828751Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.543657Z","title":"Mmlu-pro: A more robust and challenging multi-task language understanding benchmark","venue":null,"work_id":"e1f30621-8a1b-4d02-8ee7-1db177016e27","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.832809Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:d1ecd76f16453b85b2faf353f048bcd0c20f077c1d39f413ab7ec0b81fb8a522","observation_id":"77f7bc9b-f90c-4732-aa86-290f9a53859c","resolution":{"observed_at":"2026-08-15T17:07:20.548388Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21819","last_updated":"2025-06-21T08:39:06Z","snapshot_observed_at":"2026-08-14T13:23:53.106970Z","submitted_at":"2024-10-29T07:42:18Z","title":"Self-Preference Bias in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21819","snapshot_observed_at":"2026-08-15T17:07:19.836673Z","title":"Self-preference bias in llm-as-a-judge","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.836673Z"},"links":{"cited_paper":"/paper/2410.21819","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:acb74b48cfc6c95c3287eb5ee485e2d7bd7ccbe5a7c1d28b6e50776378250108","observation_id":"bbab3f8d-f82d-429a-85a0-c1756cd4f1b6","resolution":{"observed_at":"2026-08-15T17:07:19.836673Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.04368","last_updated":"2024-11-07T01:58:42Z","snapshot_observed_at":"2026-08-17T10:08:57.374438Z","submitted_at":"2024-11-07T01:58:42Z","title":"Measuring short-form factuality in large language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.04368","snapshot_observed_at":"2026-08-15T17:07:19.840580Z","title":"Measuring short-form factuality in large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.840580Z"},"links":{"cited_paper":"/paper/2411.04368","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:1d61af165b657af8d32c4462e371e90742f8623267f205bee2d58527c69514cc","observation_id":"c9f21bba-2b91-4da7-a383-e50b111deb34","resolution":{"observed_at":"2026-08-15T17:07:19.840580Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.12516","last_updated":"2025-04-16T22:27:45Z","snapshot_observed_at":"2026-08-13T03:19:29.377723Z","submitted_at":"2025-04-16T22:27:45Z","title":"BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.12516","snapshot_observed_at":"2026-08-15T17:07:19.844366Z","title":"Browsecomp: A simple yet challenging benchmark for browsing agents","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.844366Z"},"links":{"cited_paper":"/paper/2504.12516","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:156dad55193dd18f16e329a07206dfc53c5ccc60e65909d70b477c777763c548","observation_id":"18a23df0-5f8f-4c0a-9cbf-4f3bcb23d3b6","resolution":{"observed_at":"2026-08-15T17:07:19.844366Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19314","last_updated":"2025-04-18T19:36:00Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-06-27T16:47:42Z","title":"LiveBench: A Challenging, Contamination-Limited LLM Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19314","snapshot_observed_at":"2026-08-15T17:07:19.848296Z","title":"Livebench: A challenging, contamination-free llm benchmark","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.848296Z"},"links":{"cited_paper":"/paper/2406.19314","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:80b65e3eefb9661759d807873c7b659d390e7e5ea5e114ff973e81829e7b1f42","observation_id":"b7d7cb80-05cc-4274-92e7-bc59da4121d1","resolution":{"observed_at":"2026-08-15T17:07:19.848296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.11436","last_updated":"2024-06-18T04:41:07Z","snapshot_observed_at":"2026-08-16T14:17:40.986020Z","submitted_at":"2024-02-18T03:10:39Z","title":"Pride and Prejudice: LLM Amplifies Self-Bias in Self-Refinement","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.11436","snapshot_observed_at":"2026-08-15T17:07:19.852288Z","title":"Pride and prejudice: Llm amplifies self-bias in self-refinement","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.852288Z"},"links":{"cited_paper":"/paper/2402.11436","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:268494c77fd1113a028a8a1a6cd263622c29e1f1b0720db5887c88b7765dda9b","observation_id":"7ecb45b0-6194-47f7-924f-ae0483dd783e","resolution":{"observed_at":"2026-08-15T17:07:19.852288Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:19.856397Z","title":"Jimenez, Alex L","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.856397Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:7c9f8c2eab3d9785e06c25eb0fcbe7c6b38ed30d1378bbae5c182f5c3f6f1f4f","observation_id":"922a6fd3-f114-4d7b-bcbc-f62e11b2bddc","resolution":{"observed_at":"2026-08-15T17:07:19.856397Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12045","last_updated":"2024-06-17T19:33:08Z","snapshot_observed_at":"2026-08-17T20:31:29.818313Z","submitted_at":"2024-06-17T19:33:08Z","title":"$\\tau$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12045","snapshot_observed_at":"2026-08-15T17:07:19.859994Z","title":"tau -bench: A benchmark for tool-agent-user interaction in real-world domains","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.859994Z"},"links":{"cited_paper":"/paper/2406.12045","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:43d993b2fc854de776261ec152a94e1364236fa887ea80c6a5c5b0d6c55275a7","observation_id":"ce5b0cba-cc9a-4e86-bbe2-5c0fc907b01e","resolution":{"observed_at":"2026-08-15T17:07:19.859994Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02736","last_updated":"2024-10-04T03:57:47Z","snapshot_observed_at":"2026-08-01T08:21:19.528254Z","submitted_at":"2024-10-03T17:53:30Z","title":"Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02736","snapshot_observed_at":"2026-08-15T17:07:19.863897Z","title":"Justice or prejudice? quantifying biases in llm-as-a-judge","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.863897Z"},"links":{"cited_paper":"/paper/2410.02736","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:b22edc7319ec6f16bf793bbfbaea96f8179d8aca52b2c3e507be4a2c864af100","observation_id":"4f237b55-1857-4f1e-adad-db3377df5413","resolution":{"observed_at":"2026-08-15T17:07:19.863897Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1905.07830","last_updated":"2019-05-19T23:57:23Z","snapshot_observed_at":"2026-08-15T09:37:44.321271Z","submitted_at":"2019-05-19T23:57:23Z","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.07830","snapshot_observed_at":"2026-08-15T17:07:19.868052Z","title":"Hellaswag: Can a machine really finish your sentence? arXiv preprint arXiv:1905.07830 , 2019","venue":null,"work_id":null,"year":1905},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.868052Z"},"links":{"cited_paper":"/paper/1905.07830","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:3c4a1ded4e092eef81beb43490356d8e0b1509635c1a831dd4e19fd98305b6cd","observation_id":"66825604-8add-46ae-9958-6f17510d1834","resolution":{"observed_at":"2026-08-15T17:07:19.868052Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.524728Z","title":"A careful examination of large language model performance on grade school arithmetic","venue":null,"work_id":"7a809db0-6a4b-4953-91a1-11f16aa9be4d","year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.872281Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:8865dad5bf0797e4ad13c259ffabfb9a2bbdb23cc4537a4ed14664b0dc155721","observation_id":"fd6385f5-84a6-49e2-88ce-eee3f128ed90","resolution":{"observed_at":"2026-08-15T17:07:20.529186Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.04363","last_updated":"2024-12-05T17:22:04Z","snapshot_observed_at":"2026-08-12T01:29:04.014466Z","submitted_at":"2024-12-05T17:22:04Z","title":"Challenges in Trustworthy Human Evaluation of Chatbots","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.04363","snapshot_observed_at":"2026-08-15T17:07:19.876480Z","title":"Challenges in trustworthy human evaluation of chatbots","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.876480Z"},"links":{"cited_paper":"/paper/2412.04363","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:890cb7ca876df82091c969bc960b6074c27d0086a3a6ee9eb59bbbd539ccd6bf","observation_id":"7adaf2fb-c55c-4003-be7f-b533fa00d918","resolution":{"observed_at":"2026-08-15T17:07:19.876480Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-15T17:07:19.880787Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.880787Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:1a8296a2492a8b01980db017278f45b78a2878d265d9bce91ea0adbb7d62de0d","observation_id":"2fe1d022-d3b2-45cf-8ff6-2d05aac7e3ba","resolution":{"observed_at":"2026-08-15T17:07:19.880787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.06364","last_updated":"2023-09-18T14:23:02Z","snapshot_observed_at":"2026-08-07T12:51:50.861720Z","submitted_at":"2023-04-13T09:39:30Z","title":"AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.06364","snapshot_observed_at":"2026-08-15T17:07:19.885288Z","title":"Agieval: A human-centric benchmark for evaluating foundation models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.885288Z"},"links":{"cited_paper":"/paper/2304.06364","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:f8d6e805f3c90966f1cf894c657358ee874523c0e6f3e8aea0dc99fadb010dcd","observation_id":"5dde716a-4b75-4d50-b822-de3b1a0e9cca","resolution":{"observed_at":"2026-08-15T17:07:19.885288Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.11206","last_updated":"2023-05-18T17:45:22Z","snapshot_observed_at":"2026-08-08T19:18:06.171048Z","submitted_at":"2023-05-18T17:45:22Z","title":"LIMA: Less Is More for Alignment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.11206","snapshot_observed_at":"2026-08-15T17:07:19.889330Z","title":"Lima: Less is more for alignment","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.889330Z"},"links":{"cited_paper":"/paper/2305.11206","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:896139eacd5d8846b596f414170e52150fb1403be7db9b20c011da579c4483c5","observation_id":"73bd8a55-7730-41ab-a739-ca47bfd026d9","resolution":{"observed_at":"2026-08-15T17:07:19.889330Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.21493","last_updated":"2025-05-27T17:56:27Z","snapshot_observed_at":"2026-08-13T18:42:34.464744Z","submitted_at":"2025-05-27T17:56:27Z","title":"Reinforcing General Reasoning without Verifiers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.21493","snapshot_observed_at":"2026-08-15T17:07:19.893401Z","title":"Reinforcing general reasoning without verifiers","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.893401Z"},"links":{"cited_paper":"/paper/2505.21493","citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:fb81abba2333d6d09d347bc6d77786b54f77461c96243d928969cf634233b805","observation_id":"c801d62d-bb02-4a36-ab60-326a1d65e71a","resolution":{"observed_at":"2026-08-15T17:07:19.893401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:07:20.511503Z","title":"Bigcodebench: Benchmarking code generation with diverse function calls and complex instructions, 2025","venue":null,"work_id":"7280a348-e775-49d1-a6bd-b0d3622b904f","year":2025},"citing_paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions","version":1},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-15T17:07:19.897165Z"},"links":{"citing_paper":"/paper/2508.17580"},"observation_digest":"sha256:f0130c6b3f2f37de0baf769a7003d8f2dac0078b9749a4d3418a1f877f3c61f4","observation_id":"1adc20b5-9a0a-4593-926a-6c1846e92381","resolution":{"observed_at":"2026-08-15T17:07:20.516012Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2508.17580","last_updated":"2025-08-25T01:07:59Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-17T14:10:52.413869Z","submitted_at":"2025-08-25T01:07:59Z","title":"UQ: Assessing Language Models on Unsolved Questions"},"reference_resolution":{"displayed":70,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":46,"verified_exact":1,"verified_fuzzy":23},"total_outbound_references":70},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 70 of 70 outbound references and 1 inbound Pith citation observation for arXiv:2508.17580."}