{"as_of":"2026-08-07T20:08:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:824b45a525c07d7417bfc4e25422aa187ce13c8fd583899ba673da25c8952252","coverage":[{"denominator":12,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":12,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-12T09:42:05.329537Z","state":"measured"},{"denominator":13,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":13,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T04:09:21.261237Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-07T04:09:21.591895Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"cited_work":{"arxiv_id":"2607.02577","doi":null,"metadata_source":"pith","pith_arxiv_id":"2607.02577","snapshot_observed_at":"2026-08-07T04:09:21.591895Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","venue":"cs.SE","work_id":"9ca76e30-4527-40e3-991a-9f50f4c9276d","year":2026},"citing_paper":{"arxiv_id":"2608.06370","last_updated":"2026-08-06T17:58:32Z","snapshot_observed_at":"2026-08-07T19:29:20.502485Z","submitted_at":"2026-08-06T17:58:32Z","title":"The Bitter Lesson of Tool Calling","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T04:09:21.261237Z"},"links":{"cited_paper":"/paper/2607.02577","citing_paper":"/paper/2608.06370"},"observation_digest":"sha256:c3fe2b4a042467130434296a0b5251807c556fc8366cbb6acdd4c7594becc615","observation_id":"339d6ca2-2373-4c46-a921-acc5a25c0029","resolution":{"observed_at":"2026-08-07T04:09:21.711324Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2607.02577/citation-record","integrity":"/paper/2607.02577/integrity","json":"/paper/2607.02577/citation-record.json","paper":"/paper/2607.02577"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2403.07718","last_updated":"2024-07-23T06:19:28Z","snapshot_observed_at":"2026-08-02T12:09:24.340284Z","submitted_at":"2024-03-12T14:58:45Z","title":"WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07718","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Workarena: How capable are web agents at solving common knowledge work tasks?arXiv preprint arXiv:2403.07718,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2403.07718","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:0cdfb23f69118d7870a4f79d8b3d7c897931cc0e674b2ecdf5c0c7e6b631905f","observation_id":"cc4f18c1-eac5-4b46-b96e-87bef83d60ae","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04475","last_updated":"2025-03-10T09:27:03Z","snapshot_observed_at":"2026-07-06T17:56:23.317089Z","submitted_at":"2024-04-06T02:29:02Z","title":"Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.04475","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Hashimoto","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2404.04475","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:30dd8ebcb595cc3b106186dbe1c40dfc9e0ef5311756bdde3eebca214559b032","observation_id":"0c5b989e-97e3-409d-916b-7f9f9598ef9e","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2605.30738","last_updated":"2026-05-29T02:02:31Z","snapshot_observed_at":"2026-08-01T17:11:51.059319Z","submitted_at":"2026-05-29T02:02:31Z","title":"MAVEN: Improving Generalization in Agentic Tool Calling","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2605.30738","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Gorilla Team","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2605.30738","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:3f333bb4e5cdb2768805473a5728836799b19cdf2115b010d79ea1dba5c008e5","observation_id":"2ed10347-f040-4e56-9800-24b1fe5ca8d7","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Stabletoolbench: Towards stable large-scale benchmarking on tool learning of large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:9adc3b163f058b79619886ace8a9d03f69d3cc868a20090e852a8e1daf37a6eb","observation_id":"831b7501-61af-4166-8123-e3d52a3c65b3","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.08244","last_updated":"2023-10-25T06:54:12Z","snapshot_observed_at":"2026-08-02T00:07:12.855748Z","submitted_at":"2023-04-14T14:05:32Z","title":"API-Bank: A Comprehensive Benchmark for Tool-Augmented LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.08244","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Api-bank: A comprehensive benchmark for tool-augmented llms.arXiv preprint arXiv:2304.08244, 2023a","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2304.08244","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:f1b056022a12302a20e76b711b9587fbfd0e2b0496256f995103a8b16760d2fd","observation_id":"ba1ca775-7917-43df-b323-7d70bdee1b3b","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.07982","last_updated":"2025-06-09T17:52:18Z","snapshot_observed_at":"2026-07-06T21:39:13.304260Z","submitted_at":"2025-06-09T17:52:18Z","title":"$\\tau^2$-Bench: Evaluating Conversational Agents in a Dual-Control Environment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.07982","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Livemcpbench: Evaluating language agents on real mcp tool ecosystems.arXiv preprint arXiv:2506.07982,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2506.07982","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:5e01e9f294471f91f38b6cc6112ec778274f71fbe8774d5a20e97ff1e146e410","observation_id":"467de242-e166-47fe-9dd5-12bb60678a62","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15334","last_updated":"2023-05-24T16:48:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-05-24T16:48:11Z","title":"Gorilla: Large Language Model Connected with Massive APIs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15334","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Gorilla: Large language model connected with massive apis.arXiv preprint arXiv:2305.15334,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2305.15334","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:718d527aff3ada10a6cfa8d1c5477aa8e47884d7a1ebb500ded310b41155684b","observation_id":"8ab01533-22df-4547-b45e-b6bda51b857b","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05301","last_updated":"2023-09-07T12:20:45Z","snapshot_observed_at":"2026-07-06T15:40:20.267344Z","submitted_at":"2023-06-08T15:46:32Z","title":"ToolAlpaca: Generalized Tool Learning for Language Models with 3000 Simulated Cases","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05301","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Toolalpaca: Generalized tool learning for language models.arXiv preprint arXiv:2306.05301,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2306.05301","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:3f0899ca32ec5a9ac23b5021b6b55d9a5e0fcb18eb5d44385d24497245f2a373","observation_id":"ab7a93ac-9bd6-4d03-a5d9-60773118ae37","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.07972","last_updated":"2024-05-30T08:55:12Z","snapshot_observed_at":"2026-08-07T10:10:50.201348Z","submitted_at":"2024-04-11T17:56:05Z","title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.07972","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Osworld: Benchmarking multimodal agents for open-ended tasks in real computer environments.arXiv preprint arXiv:2404.07972,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2404.07972","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:a08e74f168ccafa7baa6a0df45412fd3810e17563dfc24dfa52134802a4abc84","observation_id":"3f7d5ac6-3a9a-4c0d-99d6-7867b997d14b","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.16789","last_updated":"2023-10-03T14:45:48Z","snapshot_observed_at":"2026-07-06T16:00:46.542753Z","submitted_at":"2023-07-31T15:56:53Z","title":"ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.16789","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Toolbench: Towards comprehensive, automatic and scalable evaluation for tool learning of large language models.arXiv preprint arXiv:2307.16789,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2307.16789","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:5dba23aa2cc55b2f8043bd85a24477f73a762da4effe88068fcd8fc42ce53433","observation_id":"76ea41cb-692c-4f97-bab5-d1455cbff6ae","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12045","last_updated":"2024-06-17T19:33:08Z","snapshot_observed_at":"2026-08-02T22:19:29.043854Z","submitted_at":"2024-06-17T19:33:08Z","title":"$\\tau$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12045","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Tau-bench: A benchmark for tool-agent-user interaction in real-world domains","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2406.12045","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:b405b53db2b65db517c9fa9e7eefcdd027ab6330a575dda9709b7f9fdde086e3","observation_id":"95f9596d-ada6-4c64-9c0e-85603c1955c0","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.13854","last_updated":"2024-04-16T15:13:18Z","snapshot_observed_at":"2026-08-06T12:47:01.809185Z","submitted_at":"2023-07-25T22:59:32Z","title":"WebArena: A Realistic Web Environment for Building Autonomous Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.13854","snapshot_observed_at":"2026-07-12T09:42:05.329537Z","title":"Webarena: A realistic web environment for building autonomous agents.arXiv preprint arXiv:2307.13854,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-07-12T09:42:05.329537Z"},"links":{"cited_paper":"/paper/2307.13854","citing_paper":"/paper/2607.02577"},"observation_digest":"sha256:11c21e6f60d11b3131c9280a3243f789cf05706b072a132ad28f85ef46ff541b","observation_id":"1fbfa584-db36-476a-8a56-f138926b5c25","resolution":{"observed_at":"2026-07-12T09:42:05.329537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.02577","last_updated":"2026-06-30T21:10:42Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-07T06:54:53.060230Z","submitted_at":"2026-06-30T21:10:42Z","title":"Benchmarking the Benchmarks: A Validity Audit of Tool-Calling Evaluation"},"reference_resolution":{"displayed":12,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":12,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":12},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 7 August 2026, this Paper Citation Record lists 12 of 12 outbound references and 1 inbound Pith citation observation for arXiv:2607.02577."}