{"as_of":"2026-08-18T16:38:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:1cf5d9e18bc8a24378b333244587e55c9d123b5dac15a5945a8d0bd95e1f9ee5","coverage":[{"denominator":29,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":29,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T15:31:50.457893Z","state":"measured"},{"denominator":29,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":29,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-18T06:34:40.430872+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.24268/citation-record","integrity":"/paper/2607.24268/integrity","json":"/paper/2607.24268/citation-record.json","paper":"/paper/2607.24268"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.831999Z","title":"G-eval: NLG evaluation using GPT-4 with better human alignment,","venue":null,"work_id":"2aff1acc-c63b-4972-8d4e-20944f332a7e","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.353991Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:0bf147d58c4530382bebb9c476d37a127f8fd4b0ff12f1dc4fc8c1aad28ded0e","observation_id":"d75de6bf-d70a-4e23-a4d4-afe513f9e0ae","resolution":{"observed_at":"2026-08-15T15:31:50.836135Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.820124Z","title":"Judging LLM-as-a-judge with MT-Bench and chatbot arena,","venue":null,"work_id":"db01dda9-1228-4014-b45e-d479564879e5","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.357946Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:e2baeda13274ed623894a6af7620e782f201865569c39a460ebf76ba36f61d08","observation_id":"5c66f487-d9b1-41b9-84e0-777ea94cef8f","resolution":{"observed_at":"2026-08-15T15:31:50.825044Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.809345Z","title":"Large language models are not fair evaluators,","venue":null,"work_id":"a07c36d1-3f8e-460f-89cf-e09bebc58d4d","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.361614Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:e8398422e3ee8cdd29fed124441a8ab68148ccf5a402193bb9bec3a852ea12f5","observation_id":"da6bdfa2-b864-45a8-9519-79986e161a0a","resolution":{"observed_at":"2026-08-15T15:31:50.813236Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.785144Z","title":"Length- controlled AlpacaEval: A simple way to debias automatic evaluators,","venue":null,"work_id":"4c9d2c09-a9de-4f81-b4ac-58052ee572b6","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.368490Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:92f9764cb252edcf0fd30bdf7bdd05addd129d07173f4621605ab493a7c89b65","observation_id":"fabbaf1c-095b-43ac-ae11-712faaf88e04","resolution":{"observed_at":"2026-08-15T15:31:50.789095Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.773754Z","title":"Finding blind spots in evaluator LLMs with interpretable checklists,","venue":null,"work_id":"776f79af-37b5-4f69-b324-55cecefa0dc4","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.372053Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:d42de172d20ce877530949ce6bb7ec4be213813fec93e857428c432abc5f14ac","observation_id":"8a2af24b-4d52-49a7-ad76-ac48d2d73ca4","resolution":{"observed_at":"2026-08-15T15:31:50.777598Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15594","last_updated":"2025-10-19T10:32:43Z","snapshot_observed_at":"2026-08-13T06:43:22.011336Z","submitted_at":"2024-11-23T16:03:35Z","title":"A Survey on LLM-as-a-Judge","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15594","snapshot_observed_at":"2026-08-15T15:31:50.375706Z","title":"A survey on LLM-as-a-judge,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.375706Z"},"links":{"cited_paper":"/paper/2411.15594","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:4559d1b7aeabb3ba153f91002113b8a8f652f6c7c78eaa85a0b505f26bf1bdb0","observation_id":"40ec0a0a-e7b1-4460-838a-cfe15bb4894a","resolution":{"observed_at":"2026-08-15T15:31:50.375706Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2607.08535","last_updated":"2026-07-09T14:31:05Z","snapshot_observed_at":"2026-08-07T23:22:05.988612Z","submitted_at":"2026-07-09T14:31:05Z","title":"When the Judge Changes, So Does the Measurement: Auditing LLM-as-Judge Reliability","version":1},"cited_work":{"arxiv_id":"2607.08535","doi":null,"metadata_source":"pith","pith_arxiv_id":"2607.08535","snapshot_observed_at":"2026-08-15T15:31:50.542809Z","title":"When the Judge Changes, So Does the Measurement: Auditing LLM-as-Judge Reliability","venue":"cs.CL","work_id":"ffebe468-27bf-4c76-a386-1cedb02e3938","year":2026},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.379275Z"},"links":{"cited_paper":"/paper/2607.08535","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:aa6d132b73f3add39ee551f40bcb917f54039704f3122919244765c9fcc86fff","observation_id":"921af750-2776-453b-92cc-0d22ee0372b3","resolution":{"observed_at":"2026-08-15T15:31:50.549766Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.762405Z","title":"Self- refine: Iterative refinement with self-feedback,","venue":null,"work_id":"c90daace-6383-4e2e-86e2-6454beab50c3","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.383087Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:b52ae6d99a7d7bc52130dcf95648429f6f2cd77898cbc360f9f4ff4574c7ec0d","observation_id":"dd879589-7aaf-4a69-bf39-ee792da652bb","resolution":{"observed_at":"2026-08-15T15:31:50.766115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.750197Z","title":"Reflexion: Language agents with verbal reinforcement learning,","venue":null,"work_id":"602b548e-719b-46da-b294-77fea8ef46bb","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.387492Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:a3ae81b5e8cc8931c9fc3bc0512e1d9b344c14a08fe42043f85cd018b1ff2a02","observation_id":"b28cc601-9f98-4768-9380-afaa294fa5f1","resolution":{"observed_at":"2026-08-15T15:31:50.754828Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.739106Z","title":"Large language models cannot self- correct reasoning yet,","venue":null,"work_id":"04e53892-50ff-4461-98f3-be82c069c482","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.391256Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:045d8c951655268b8a32460bf55bd8c7c999e409563cf21892559cb8272fb3a9","observation_id":"ca42c752-0fd2-4167-9895-e64840db8dd4","resolution":{"observed_at":"2026-08-15T15:31:50.743071Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.726316Z","title":"Large language models can self-correct with key condition verification,","venue":null,"work_id":"3d300604-11df-4162-91a3-713da87e8709","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.394873Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:68ce3a70b3a73ad4ce5b8a7841f4c1c2f442ad1d7e7064392132bbdeaa4be6eb","observation_id":"07c5eeca-5556-4117-8535-0fc07ae76568","resolution":{"observed_at":"2026-08-15T15:31:50.730242Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.712844Z","title":"When can LLMs actually correct their own mistakes? a critical survey of self-correction of LLMs,","venue":null,"work_id":"d2d1efd1-60f4-408b-97ec-a818d5ce46cc","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.398445Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:4215e2afee5123d3a30d1772626fa40cd461e19eea591d2b529176101ad40ff2","observation_id":"3642869e-3a41-4ea7-9962-526b811e5a2f","resolution":{"observed_at":"2026-08-15T15:31:50.718264Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.700094Z","title":"Automatically correcting large language models: Surveying the landscape of diverse automated correction strategies,","venue":null,"work_id":"358a9ef6-4883-4d2f-b494-ad007ba440d3","year":null},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.402226Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:6d52d8442cb661ec64a5a43f060dfb2eb5e9d63911fced191e21e65aaf07270e","observation_id":"3080dec1-efe7-451b-90ad-42d5e4b8a65c","resolution":{"observed_at":"2026-08-15T15:31:50.704086Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.675942Z","title":"Chain-of-thought prompting elicits reasoning in large language models,","venue":null,"work_id":"890547db-aa2c-480f-9022-8382001aa695","year":2022},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.409957Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:0b9afdea8467f8ebe3eff9c82ea2d688c535743bd62828cd8c18761f398123bf","observation_id":"25d25750-2f72-4bb9-b2c9-04581b52e961","resolution":{"observed_at":"2026-08-15T15:31:50.680104Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.413411Z","title":"Self-consistency improves chain of thought reasoning in language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.413411Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:ac1f4765a5c059a2a4b6339a385b8a11bc0baa9e1e219afa79915bd38fc79a8b","observation_id":"ade4312d-30e1-414d-b895-5a1a88b5dd4a","resolution":{"observed_at":"2026-08-15T15:31:50.413411Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.657547Z","title":"Scaling LLM test-time compute optimally can be more effective than scaling parameters for reasoning,","venue":null,"work_id":"2c5db892-7a2a-43b0-bb67-33790b518d8f","year":2025},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.416852Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:1921c1ca9814a09f0f90be1edfe3de830a1f11af6d7cceaa0fca62ca10921bb0","observation_id":"50114e22-3636-4e88-885f-dd3bd2c83f94","resolution":{"observed_at":"2026-08-15T15:31:50.661638Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02725","last_updated":"2024-10-03T17:47:29Z","snapshot_observed_at":"2026-08-18T16:38:39.590606Z","submitted_at":"2024-10-03T17:47:29Z","title":"Adaptive Inference-Time Compute: LLMs Can Predict if They Can Do Better, Even Mid-Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02725","snapshot_observed_at":"2026-08-15T15:31:50.420577Z","title":"Adaptive inference-time compute: LLMs can predict if they can do better, even mid- generation,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.420577Z"},"links":{"cited_paper":"/paper/2410.02725","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:993a93fbb4d03f41b6a9423cac414c3380db05a6b09d0e7342c911cda219a630","observation_id":"7314de4d-bd1f-4887-a915-23c7ca31f8c5","resolution":{"observed_at":"2026-08-15T15:31:50.420577Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.19393","last_updated":"2025-03-01T06:07:39Z","snapshot_observed_at":"2026-08-17T11:00:39.333660Z","submitted_at":"2025-01-31T18:48:08Z","title":"s1: Simple test-time scaling","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.19393","snapshot_observed_at":"2026-08-15T15:31:50.424448Z","title":"s1: Simple test-time scaling,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.424448Z"},"links":{"cited_paper":"/paper/2501.19393","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:f53969db42ddde80c1cbdb61d90fb9dc5c1f4b905a5cc833dc0adbea9b634c35","observation_id":"ecb34dfa-cb8b-4ab3-b587-21017041c133","resolution":{"observed_at":"2026-08-15T15:31:50.424448Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.645969Z","title":"Beyond accuracy: Behavioral testing of NLP models with CheckList,","venue":null,"work_id":"d4304fbb-3ea7-4d28-a858-0dbf71189a10","year":2020},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.428160Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:e509fd97d573d261c52516a307ca1c96e336c705dc0bf191e824211284dc45ac","observation_id":"6ca74e2b-4fb4-46fa-a8d7-80406f20dc76","resolution":{"observed_at":"2026-08-15T15:31:50.650193Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.631219Z","title":"Evaluating evaluation metrics: A framework for analyzing NLG evaluation metrics using measurement theory,","venue":null,"work_id":"e36a683f-3d4d-4d2f-b810-cd33220133e8","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.432593Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:fc72e5fec88df36318ba5724ccd6cdeebb7d3b6e8a0a49f9ab8c9727e079c44e","observation_id":"bcc1da1e-bebe-43fd-a055-9858bb3c3b5d","resolution":{"observed_at":"2026-08-15T15:31:50.635562Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.617018Z","title":"Capturing failures of large language models via human cognitive biases,","venue":null,"work_id":"29900739-ca27-4da2-bb78-656a9cb5e166","year":2022},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.436117Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:c04f8416c3e51bd003fcce433b3b839a01ed2a6776d12701d9d2d3624470972a","observation_id":"0a34877f-8e1b-4069-bf9f-148d9abeaedb","resolution":{"observed_at":"2026-08-15T15:31:50.621651Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.603810Z","title":"What will it take to fix benchmarking in natural language understanding?","venue":null,"work_id":"9e706b4b-0570-408b-a0e0-7db8de6a8dcb","year":2021},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.439905Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:f4654d5541108c61133d9b10517afce537ee4954608c24c605acf5d80543f264","observation_id":"83e14cfb-c70a-4b60-a2a3-aca1262bb7c7","resolution":{"observed_at":"2026-08-15T15:31:50.607959Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.591638Z","title":"A systematic classification of knowledge, reasoning, and context within the ARC dataset,","venue":null,"work_id":"c5147751-6318-4190-acdb-9479e9ca65ed","year":2018},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.443972Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:00d17adad1c99ee2977f93eed7ac95fcfa0bdee579e41acc59f3b8625dd5ebdb","observation_id":"28797bff-1bd2-442a-b0e6-6e089837288d","resolution":{"observed_at":"2026-08-15T15:31:50.595915Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.580393Z","title":"Towards a consensus taxonomy for annotating errors in automatically generated text,","venue":null,"work_id":"a0e845cc-b7cd-456e-af5c-59c8d59245d2","year":2023},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.447539Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:f155b9f5103d446aa53a2d20ed89c5fdeb6c0d9e04e962a283a59a29dbeb986c","observation_id":"48cfcb7d-00fa-4ef9-9057-4cc377d49652","resolution":{"observed_at":"2026-08-15T15:31:50.584621Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.569600Z","title":"Measuring mathematical problem solving with the MATH dataset,","venue":null,"work_id":"78040e60-dad7-40a3-92d7-d3f22fb712af","year":2021},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.450911Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:55bf171a0497024189ea818f0a1c771505e2bf976eb580ecfe4ab0f8622b26c6","observation_id":"fd2e61cb-a871-42f3-8500-ee89c3652bc7","resolution":{"observed_at":"2026-08-15T15:31:50.573378Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-08-14T19:36:07.505691Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-15T15:31:50.454276Z","title":"Think you have solved question answering? try ARC, the AI2 reasoning challenge,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.454276Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:182e4ba29627c38691a1c7ab90fd44082a06e7fc04658dde9a84abbea01a51a9","observation_id":"ff5c7b8b-98bb-484a-98fd-010581f035ab","resolution":{"observed_at":"2026-08-15T15:31:50.454276Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.09388","last_updated":"2025-05-14T13:41:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-05-14T13:41:34Z","title":"Qwen3 Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.09388","snapshot_observed_at":"2026-08-15T15:31:50.457893Z","title":"Qwen3 technical report,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.457893Z"},"links":{"cited_paper":"/paper/2505.09388","citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:6bc65714e13cdbf8e4a8e33d7145b57cbc815343e6bed612c45ccb896fe1eded","observation_id":"41c5571a-5ef5-4610-a38d-46573890dddd","resolution":{"observed_at":"2026-08-15T15:31:50.457893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.688411Z","title":"Available: https://aclanthology.org/2024.tacl-1.27/","venue":null,"work_id":"9006b7d1-79cb-417d-a21d-4b8a1496e2e5","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.406226Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:e25525d92c22994958f4e40488b73f59f2f823df1b3da0f6f1fadd9ed9ca2075","observation_id":"efda8ccd-5b59-43e6-937c-45ba78e7daeb","resolution":{"observed_at":"2026-08-15T15:31:50.692569Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:31:50.796460Z","title":"Available: https://aclanthology.org/2024.acl-long.511/","venue":null,"work_id":"01e59057-8c5d-4ea7-99ef-fcc91e212fa3","year":2024},"citing_paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets","version":2},"reference_index":9450,"source":"pdf_text","source_observed_at":"2026-08-15T15:31:50.365254Z"},"links":{"citing_paper":"/paper/2607.24268"},"observation_digest":"sha256:7afb783f1ae5df459b7430474d731a034338aa3032caf7b53516f259e4bb2b5b","observation_id":"0f2b6586-f2b0-486d-84b9-4091edc43772","resolution":{"observed_at":"2026-08-15T15:31:50.800533Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2607.24268","last_updated":"2026-07-30T06:23:36Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-18T05:03:03.371194Z","submitted_at":"2026-07-27T11:04:17Z","title":"Accuracy Hides How Language Models Fail: Measuring Failure States Under Matched Output Budgets"},"reference_resolution":{"displayed":29,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":6,"verified_exact":1,"verified_fuzzy":22},"total_outbound_references":29},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 29 of 29 outbound references and 0 inbound Pith citation observations for arXiv:2607.24268."}