{"as_of":"2026-08-19T19:15:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:ce1cdb93f7ad711879e7242bdecc6dc09fa774e86c33113b066e630f0900ba34","coverage":[{"denominator":30,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":30,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T07:19:04.462120Z","state":"measured"},{"denominator":30,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":30,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02444/citation-record","integrity":"/paper/2608.02444/integrity","json":"/paper/2608.02444/citation-record.json","paper":"/paper/2608.02444"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:01.474902Z","title":"Efficient benchmarking of AI agents,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.474902Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:d73eabf1c0190df51a3c6e2b39881c49f1033c7999fcfecf3e74888ede99266f","observation_id":"6bc585d6-d3f4-4f24-a23c-d177e6a5f6aa","resolution":{"observed_at":"2026-08-04T07:19:01.474902Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:01.581554Z","title":"SWE-bench: Can language models resolve real-world GitHub issues?","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.581554Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:bb2d2c1d1a64087982ec10fdcc246a3036c26383c3953b5a917d466491e34a76","observation_id":"92eeb214-c9da-4e4e-98e0-76bf282c353a","resolution":{"observed_at":"2026-08-04T07:19:01.581554Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:01.644754Z","title":"SWE-bench leaderboards,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.644754Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:5f63aedd5ee2d14c5c6d7677b6ffcd238c6e8bd1957fda7715f544699fbd56ce","observation_id":"79952cdf-d754-4245-a16b-8bf184e87cbf","resolution":{"observed_at":"2026-08-04T07:19:01.644754Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.18901","last_updated":"2024-07-26T17:55:45Z","snapshot_observed_at":"2026-08-16T13:30:42.217868Z","submitted_at":"2024-07-26T17:55:45Z","title":"AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.18901","snapshot_observed_at":"2026-08-04T07:19:01.792257Z","title":"AppWorld: A controllable world of apps and people for benchmarking interactive coding agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.792257Z"},"links":{"cited_paper":"/paper/2407.18901","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:d79618e3fd3459ce8f9db8054593b30555da3a9ae641b52a9a98730b5209e219","observation_id":"5ccd03e1-e641-4d5b-ae85-d74efbe8308e","resolution":{"observed_at":"2026-08-04T07:19:01.792257Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.07972","last_updated":"2024-05-30T08:55:12Z","snapshot_observed_at":"2026-08-14T22:26:00.902198Z","submitted_at":"2024-04-11T17:56:05Z","title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.07972","snapshot_observed_at":"2026-08-04T07:19:01.939210Z","title":"OSWorld: Benchmarking multimodal agents for open-ended tasks in real computer environments,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.939210Z"},"links":{"cited_paper":"/paper/2404.07972","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:a4db257047a5891c045de64306e3e5ff198926a4b96e4869915691f4961850e9","observation_id":"1b8568ab-4219-4297-b317-7b68302d4ed2","resolution":{"observed_at":"2026-08-04T07:19:01.939210Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12045","last_updated":"2024-06-17T19:33:08Z","snapshot_observed_at":"2026-08-17T20:31:29.818313Z","submitted_at":"2024-06-17T19:33:08Z","title":"$\\tau$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12045","snapshot_observed_at":"2026-08-04T07:19:02.008966Z","title":"τ-bench: A benchmark for tool-agent-user interaction in real-world domains,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.008966Z"},"links":{"cited_paper":"/paper/2406.12045","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:410f39e3f9d8761639c5dd588bc9a4f884fc72f3753a8bed56b71a0ab8ecbe05","observation_id":"74d1fca5-746a-4604-8d0c-8c63bee87aec","resolution":{"observed_at":"2026-08-04T07:19:02.008966Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.07982","last_updated":"2025-06-09T17:52:18Z","snapshot_observed_at":"2026-08-14T06:34:01.114459Z","submitted_at":"2025-06-09T17:52:18Z","title":"$\\tau^2$-Bench: Evaluating Conversational Agents in a Dual-Control Environment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.07982","snapshot_observed_at":"2026-08-04T07:19:02.134876Z","title":"τ 2-Bench: Evaluating conversational agents in a dual-control environment,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.134876Z"},"links":{"cited_paper":"/paper/2506.07982","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:7d12ae4b0468d143eb4b33fcfa287e9f02dbb2a6624ac56085eff03381f79526","observation_id":"3f7c8a70-366e-4211-9552-44826efe4c5d","resolution":{"observed_at":"2026-08-04T07:19:02.134876Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2601.11868","last_updated":"2026-01-17T01:29:30Z","snapshot_observed_at":"2026-07-06T22:41:58.373427Z","submitted_at":"2026-01-17T01:29:30Z","title":"Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2601.11868","snapshot_observed_at":"2026-08-04T07:19:02.235500Z","title":"Terminal-Bench: Benchmarking agents on hard, realistic tasks in command line interfaces,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.235500Z"},"links":{"cited_paper":"/paper/2601.11868","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:79f318f84390576a3bcddce62627316aa36cc8790295d40c4f1e07229e483fe3","observation_id":"3cc61c9a-68e1-4bcb-9572-d3e069f682ba","resolution":{"observed_at":"2026-08-04T07:19:02.235500Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.346033Z","title":"Holistic Agent Leaderboard: The missing infrastructure for AI agent evaluation,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.346033Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:63e62509c191c18f8f744e435abe3f5ab5ef4411281844d3ad8e5c05690b2848","observation_id":"8b46a2fb-923c-4a78-afb6-ffb4da4e48dc","resolution":{"observed_at":"2026-08-04T07:19:02.346033Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.458781Z","title":"Sequential tests of statistical hypotheses,","venue":null,"work_id":null,"year":1945},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.458781Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:f10421a2bb94b0f9b6b0949e3adc84f3cf6793d62dc3a873b16b7da7400902da","observation_id":"734ac525-f565-4975-ae5d-2ef05c0342ce","resolution":{"observed_at":"2026-08-04T07:19:02.458781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.564760Z","title":"Time-uniform, nonparametric, nonasymptotic confidence sequences,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.564760Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:a476e00011c9dbdda71dc3d813e5fb04cc4d58cb69d9a5532c4c9ec7a8a67124","observation_id":"6431829b-6e8f-4f81-b69c-a24188277490","resolution":{"observed_at":"2026-08-04T07:19:02.564760Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.684572Z","title":"Selectivenet: A deep neural network with an integrated reject option,","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.684572Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:289b9611608f2402c05f5af3066d7793aa8ed787fe4025743a121b042e6bde53","observation_id":"a94a15f9-e1c9-4cf2-bcac-fa24abe10bc4","resolution":{"observed_at":"2026-08-04T07:19:02.684572Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.781995Z","title":"Consistent estimators for learning to defer to an expert,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.781995Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:7923e69317e6c840ea9e6b953dde7898f9e736463c2ece6b56899dfab7f4499c","observation_id":"39e41e85-b040-4894-9449-735bd43a3d09","resolution":{"observed_at":"2026-08-04T07:19:02.781995Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2208.02814","last_updated":"2025-06-13T18:38:23Z","snapshot_observed_at":"2026-08-19T13:28:01.411212Z","submitted_at":"2022-08-04T17:59:44Z","title":"Conformal Risk Control","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.02814","snapshot_observed_at":"2026-08-04T07:19:02.947149Z","title":"Conformal risk control,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.947149Z"},"links":{"cited_paper":"/paper/2208.02814","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:a85a39f8a4e461a03bdd205053fa676c695cd335e908f1f5cc53a18c03d80f82","observation_id":"3ffb8890-b1a4-49d8-bc03-2ecdbb92b577","resolution":{"observed_at":"2026-08-04T07:19:02.947149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:03.030711Z","title":"Holistic evaluation of language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.030711Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:a68137eed7811eca598cd48191d7b43e1f4fa284ede9a897e8eab55a7734321e","observation_id":"7792a26e-f3aa-4191-abc4-f47f294f57c8","resolution":{"observed_at":"2026-08-04T07:19:03.030711Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:03.152718Z","title":"Judging LLM-as-a-judge with MT-Bench and Chatbot Arena,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.152718Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:8c16d42403a6f0c9bf6e77e4cbfbb6cd8173f648aa339c72027272ad9485dcaf","observation_id":"8d58f2f3-9dca-4162-ac89-0903f8683b9c","resolution":{"observed_at":"2026-08-04T07:19:03.152718Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.03814","last_updated":"2025-05-02T17:05:01Z","snapshot_observed_at":"2026-08-17T15:11:12.016051Z","submitted_at":"2025-05-02T17:05:01Z","title":"Cer-Eval: Certifiable and Cost-Efficient Evaluation Framework for LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.03814","snapshot_observed_at":"2026-08-04T07:19:03.239193Z","title":"Cer-Eval: Certifiable and cost-efficient evaluation framework for LLMs,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.239193Z"},"links":{"cited_paper":"/paper/2505.03814","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:f6e7a335432f3efc8bec6e38adba975c0c07184ad1d78bdb031ed6f2bd9ff06f","observation_id":"97598b0f-a068-43aa-84ad-40580d288ceb","resolution":{"observed_at":"2026-08-04T07:19:03.239193Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2601.20251","last_updated":"2026-05-08T20:35:11Z","snapshot_observed_at":"2026-08-15T00:48:56.572840Z","submitted_at":"2026-01-28T04:59:20Z","title":"Efficient Evaluation of LLM Performance with Statistical Guarantees","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2601.20251","snapshot_observed_at":"2026-08-04T07:19:03.321698Z","title":"Efficient evaluation of LLM per- formance with statistical guarantees,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.321698Z"},"links":{"cited_paper":"/paper/2601.20251","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:4340dc18ccd99a2f47818def3b063261a940e26ef323e7697193e171dd31571b","observation_id":"7499cc94-7d84-4097-bd2b-d41870d8d26f","resolution":{"observed_at":"2026-08-04T07:19:03.321698Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:03.447337Z","title":"Probability inequalities for the sum in sampling without replacement,","venue":null,"work_id":null,"year":1974},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.447337Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:17cebe29e5db6dfda10fec61c9b731c6404ec0201efe631efb420617ebe0bbaf","observation_id":"b31fe661-ddad-45a7-86c4-106aa6de11f4","resolution":{"observed_at":"2026-08-04T07:19:03.447337Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:03.554167Z","title":"On the two different aspects of the representative method: The method of stratified sampling and the method of purposive selection,","venue":null,"work_id":null,"year":1934},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.554167Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:57135f7065251526cc458721fb151860384e8e78db7c59cc4b95f1929cb38602","observation_id":"3f59e237-2679-4419-913c-2c0e66cc0c2f","resolution":{"observed_at":"2026-08-04T07:19:03.554167Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01502","last_updated":"2024-07-01T17:48:14Z","snapshot_observed_at":"2026-08-19T14:24:18.304488Z","submitted_at":"2024-07-01T17:48:14Z","title":"AI Agents That Matter","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01502","snapshot_observed_at":"2026-08-04T07:19:03.668280Z","title":"AI agents that matter,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.668280Z"},"links":{"cited_paper":"/paper/2407.01502","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:e2785b8e86d7d5f4448abe9fdce5ca9f2ee3a0d65754b7ab178e4cd98c05b264","observation_id":"761f7a42-9a06-4c22-945a-1902bccad3ac","resolution":{"observed_at":"2026-08-04T07:19:03.668280Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2605.21404","snapshot_observed_at":"2026-08-04T07:19:03.765535Z","title":"What twelve LLM agent benchmark papers disclose about themselves: A pilot audit and an open scoring schema,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.765535Z"},"links":{"cited_paper":"/paper/2605.21404","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:997ea6686d3e372bb65ef86dfe02d0970d6da2ded4eeb2696477a262d7963db7","observation_id":"ea9a4f83-5ac8-44db-80e7-18816f51053f","resolution":{"observed_at":"2026-08-04T07:19:03.765535Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2602.22953","last_updated":"2026-05-11T10:27:38Z","snapshot_observed_at":"2026-08-17T14:17:45.326868Z","submitted_at":"2026-02-26T12:48:02Z","title":"General Agent Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2602.22953","snapshot_observed_at":"2026-08-04T07:19:03.880384Z","title":"General agent evaluation,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.880384Z"},"links":{"cited_paper":"/paper/2602.22953","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:c619b102c0cde46c008823018e80651a4bf6936f679486c045b75e042fdc2f2f","observation_id":"511290d2-3acb-4861-a129-130a80233df0","resolution":{"observed_at":"2026-08-04T07:19:03.880384Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.03056","last_updated":"2025-03-04T23:41:02Z","snapshot_observed_at":"2026-08-16T12:53:02.860339Z","submitted_at":"2025-03-04T23:41:02Z","title":"A2Perf: Real-World Autonomous Agents Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.03056","snapshot_observed_at":"2026-08-04T07:19:04.013592Z","title":"A2Perf: Real-world autonomous agents benchmark,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:04.013592Z"},"links":{"cited_paper":"/paper/2503.03056","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:464d4bf0dcb6106ea6573b525c76fb41f8eebcfd9f658c576f5b74792d2a664e","observation_id":"72ebbd39-fd75-4c41-9c4c-640a81ef4233","resolution":{"observed_at":"2026-08-04T07:19:04.013592Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:04.129035Z","title":"ProSoftArena: Benchmarking hierarchical capabilities of multimodal agents in professional software environments,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:04.129035Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:fb02efa010729f57841bca850b9f7e9b1fc72f2eb715af46df834903c6e0edc0","observation_id":"05d92a8d-e366-44b7-87f8-f158dcc5a18a","resolution":{"observed_at":"2026-08-04T07:19:04.129035Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2509.09321","last_updated":"2025-09-11T10:10:48Z","snapshot_observed_at":"2026-08-14T23:09:12.344345Z","submitted_at":"2025-09-11T10:10:48Z","title":"Towards Adaptive ML Benchmarks: Web-Agent-Driven Construction, Domain Expansion, and Metric Optimization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2509.09321","snapshot_observed_at":"2026-08-04T07:19:04.202588Z","title":"Towards adaptive ML benchmarks: Web-agent-driven construction, domain expansion, and metric optimization,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:04.202588Z"},"links":{"cited_paper":"/paper/2509.09321","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:97b6481f9efc7ecf43254c0de6241f5caf1ba0d7c27c042b09da392665d7429f","observation_id":"c77656e4-d561-4551-af1c-1a4ce5632115","resolution":{"observed_at":"2026-08-04T07:19:04.202588Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14573","last_updated":"2025-04-06T20:37:50Z","snapshot_observed_at":"2026-08-19T11:49:14.266180Z","submitted_at":"2024-05-23T13:48:54Z","title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14573","snapshot_observed_at":"2026-08-04T07:19:04.302624Z","title":"AndroidWorld: A dynamic benchmarking environment for autonomous agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:04.302624Z"},"links":{"cited_paper":"/paper/2405.14573","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:54ffdadb843e7eb4cf43b41bc86fbc0aec2230ec869f1860b2691e5686937c9f","observation_id":"1cda3a64-7c95-415c-9335-232e8daec7c4","resolution":{"observed_at":"2026-08-04T07:19:04.302624Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2604.23853","last_updated":"2026-04-26T19:44:10Z","snapshot_observed_at":"2026-08-18T16:42:13.412164Z","submitted_at":"2026-04-26T19:44:10Z","title":"ClawTrace: Cost-Aware Tracing for LLM Agent Skill Distillation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2604.23853","snapshot_observed_at":"2026-08-04T07:19:04.462120Z","title":"ClawTrace: Cost-aware tracing for LLM agent skill distillation,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:04.462120Z"},"links":{"cited_paper":"/paper/2604.23853","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:d8f5ee641c3a1dfde0208b7f727f2467bbceb283d72e4f2f2203148ea817b029","observation_id":"831ec9eb-dd1d-4816-ab7f-e3b29418f223","resolution":{"observed_at":"2026-08-04T07:19:04.462120Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:02.865413Z","title":"7076–7087","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":119,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:02.865413Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:755e3a2d72bc4e58e1d490272ee56f0f65001cac3898c1975c692f9f0bad63a6","observation_id":"b2ea0bdf-45b3-4e53-9e15-d98befaa2c74","resolution":{"observed_at":"2026-08-04T07:19:02.865413Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T07:19:01.738273Z","title":"Available: https://www.swebench.com/","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":2026,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:01.738273Z"},"links":{"citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:3a7673d6c85be16997c88674955aa1e6d3c6403c49e3c3c7cb18f388aebf2d69","observation_id":"ce3acd5e-8496-42bb-9064-eeb2690e0555","resolution":{"observed_at":"2026-08-04T07:19:01.738273Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision"},"reference_resolution":{"displayed":30,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":30,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":30},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 30 of 30 outbound references and 0 inbound Pith citation observations for arXiv:2608.02444."}