{"as_of":"2026-08-22T19:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:218f60f4a29486a1e5117dab5f296312bd1644d47dfabcc565d00ac172e74f62","coverage":[{"denominator":23,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":23,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-21T05:20:01.498711Z","state":"measured"},{"denominator":24,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":24,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-22T06:32:14.747728+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T07:19:03.765535Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2605.21404","snapshot_observed_at":"2026-08-04T07:19:03.765535Z","title":"What twelve LLM agent benchmark papers disclose about themselves: A pilot audit and an open scoring schema,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02444","last_updated":"2026-08-03T16:22:51Z","snapshot_observed_at":"2026-08-09T23:41:48.184811Z","submitted_at":"2026-08-03T16:22:51Z","title":"ParEvalLayer: When Partial LLM-Agent Evaluations Support a Decision","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-04T07:19:03.765535Z"},"links":{"cited_paper":"/paper/2605.21404","citing_paper":"/paper/2608.02444"},"observation_digest":"sha256:997ea6686d3e372bb65ef86dfe02d0970d6da2ded4eeb2696477a262d7963db7","observation_id":"ea9a4f83-5ac8-44db-80e7-18816f51053f","resolution":{"observed_at":"2026-08-04T07:19:03.765535Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2605.21404/citation-record","integrity":"/paper/2605.21404/integrity","json":"/paper/2605.21404/citation-record.json","paper":"/paper/2605.21404"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"SWE-bench: Can Language Models Resolve Real- World GitHub Issues?","venue":null,"work_id":"dcf7476b-39c9-429a-a329-834c2f911d02","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:e7b82764e5173099127bbb448642593ca032719b114e0104d66c380a2fef8fd0","observation_id":"1319b642-c278-4345-a346-6dea9f29fee6","resolution":{"observed_at":"2026-05-21T05:24:39.780117Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Introducing SWE-bench Verified","venue":null,"work_id":"4089d444-2f04-4d23-b437-5c299e9efbd7","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:424fa44dac116f3c71501892e33499b581abd175ed741ed8d85969d049004d52","observation_id":"19dc41bb-0c6b-445e-a290-a7945118a295","resolution":{"observed_at":"2026-05-21T05:24:39.777505Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"WebArena: A Realistic Web Environment for Building Autonomous Agents","venue":null,"work_id":"e674acf9-7404-48b2-a114-b9bc4a1ff61e","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:034d45f7d61929a67c279970168526355110d14f76cb28a601daed129e472d7b","observation_id":"246da87d-6822-41d7-8959-acbeed37cf3e","resolution":{"observed_at":"2026-05-21T05:24:39.771129Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks","venue":null,"work_id":"3440d1c2-81f1-4374-b175-f7c30b3b8d8b","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:666edda43e182fd4444fc388f45fde92d0dd82aef6f72db68d868b34fb1f41cb","observation_id":"92100b1f-5529-4cc3-8027-9c8d3f0510bf","resolution":{"observed_at":"2026-05-21T05:24:39.766818Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Mind2Web: Towards a Generalist Agent for the Web","venue":null,"work_id":"a3a40576-4a29-4908-8a65-075e7d3ee213","year":2023},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:ae5912fac1277517c4576440c36c9788c9bf0df9f957e59e32895162f269277e","observation_id":"b0885c3e-0050-47f9-8b81-0ac3d0b92bd3","resolution":{"observed_at":"2026-05-21T05:24:39.768957Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"OSWorld: Benchmarking Multimodal Agents for Open- Ended Tasks in Real Computer Environments","venue":null,"work_id":"2a7674fa-136c-4559-bcad-dd16acdd206a","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:092d0f17b2658c6fffdb67ddfb8871aa612437971e03ad5c842648128b1b6fd7","observation_id":"4a2bd0a5-4fb1-4a8f-8816-e1a73e304e69","resolution":{"observed_at":"2026-05-21T05:24:39.762101Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"GAIA: A Benchmark for General AI Assistants","venue":null,"work_id":"ca20137a-ee3c-4af8-a755-d325cf1bfed0","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:c27cdfe5aabb2c38adaf35da2e86ae17bb7295ffe5ab21ec4c73464b7d9a9eff","observation_id":"988d3a46-306a-46a7-a11a-a6267d881eed","resolution":{"observed_at":"2026-05-21T05:24:39.764433Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"AgentBench: Evaluating LLMs as Agents","venue":null,"work_id":"eaec9c73-43e7-4a95-8b5c-c1ac1f1036bb","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:f7248c45192f36965337b1302817fd0e2d36cc2beb6a82694d4fb4ad80f734b7","observation_id":"b684f4ac-d9d6-4aa1-8bfc-28bf9eb51d12","resolution":{"observed_at":"2026-05-21T05:24:39.773406Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"AgentBoard: An Analytical Evaluation Board of Multi-Turn LLM Agents","venue":null,"work_id":"19c04fd1-99a9-4d6a-998e-b38a274bc41a","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:16c45922dcf1e6f50117d2a475b99d8026e3685389a2fbb49c661245431c701a","observation_id":"e0f895b7-277c-4f02-bce0-c5c94f159192","resolution":{"observed_at":"2026-05-21T05:24:39.775511Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learn- ing Engineering","venue":null,"work_id":"468aa640-8a8e-474f-8d87-e1f15e92a810","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:9a5216c0c3c219361a0102dfa02947b3aab97674d5fec18ae5aec6fc27655946","observation_id":"b54df31e-e155-4fd5-aaf7-322cf87c742d","resolution":{"observed_at":"2026-05-21T05:24:39.782088Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-20T20:50:23.483838Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":"2107.03374","doi":"10.48550/arxiv.2107.03374","metadata_source":"pith","pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Evaluating Large Language Models Trained on Code","venue":"cs.LG","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:e7a2d9ab7d18cced9302ff4c540aed5941816352fcfad890736f14bfeecccdf3","observation_id":"42b2d626-6e83-4516-a93a-221a8542bfb4","resolution":{"observed_at":"2026-05-21T05:23:58.699582Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-01T08:08:23.404839+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-01T08:08:23.404839+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-15T17:40:38.050939Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":"2108.07732","doi":"10.1007/s11390-025-5518-5","metadata_source":"pith","pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-07-11T11:50:26.030339Z","title":"Program Synthesis with Large Language Models","venue":"cs.PL","work_id":"fd241a05-03b9-4de2-9588-9d77ce176125","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:9750b0671d586e3dee36414f23e2663cfdebdd6ac701bb37cc3cfba507bd6097","observation_id":"5c0f863b-150b-40ea-b177-4aef3d3619ee","resolution":{"observed_at":"2026-05-21T05:23:58.696044Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":"2110.14168","doi":"10.1002/j.1545-","metadata_source":"pith","pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-07-11T11:50:26.030339Z","title":"Training Verifiers to Solve Math Word Problems","venue":"cs.LG","work_id":"acab1aa8-b4d6-40e0-a3ee-25341701dca2","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:a8786f123366e09a7875ed770544a8c96c5d836e2ac87706ceaff5cea9ece454","observation_id":"a0410ad2-d98a-4fa5-acbe-0db7cf0e1888","resolution":{"observed_at":"2026-05-21T05:23:58.702679Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Measuring Massive Multitask Language Understanding","venue":null,"work_id":"57177404-f68d-4c91-9c96-17269be5b604","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:bd6857b5dce081771a854cc0588f7aa0c68e4c621cd3608738c00265b9c85b5c","observation_id":"9d8a25fe-c516-4d7b-a070-db0f3027c6b7","resolution":{"observed_at":"2026-05-21T05:24:39.784014Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Deep Reinforcement Learning that Matters","venue":null,"work_id":"5ff7e2d3-5956-4790-9c62-768962d31084","year":2018},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:77bb35be836a752184697a6aa5be29cdb0e79f040aea8fecfba99973211e3321","observation_id":"72fcd949-b342-448f-a49e-b6c0afef3062","resolution":{"observed_at":"2026-05-21T05:24:39.794891Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Improving Reproducibility in Machine Learning Research","venue":null,"work_id":"68fee416-f18e-44b9-9444-cb8e4b6c9b5e","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:eae04682b9913719d6831fc28c279a34af4e081c5a45be512c67d4f860cbce00","observation_id":"8bdff259-05b9-4d1f-b6aa-822671318921","resolution":{"observed_at":"2026-05-21T05:24:39.796945Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Model Cards for Model Reporting","venue":null,"work_id":"6317c4a8-48f8-4a86-9e8c-75ed91e2ff42","year":2019},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:65fbcbb5312f210d4182db5995fd5a788ed0d7a83821f4c6efb2774cd09c1c72","observation_id":"b0fbc7ef-815b-411b-a48e-a9f54ecb7e5c","resolution":{"observed_at":"2026-05-21T05:24:39.790556Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Datasheets for Datasets","venue":null,"work_id":"06a12d37-1434-426b-8f62-a8c0449f97b1","year":2021},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:e8e502da67ba2aeee86e2441e4b291b914a1671603958a56b4a84c6e3e490319","observation_id":"d2a35313-60fa-4652-bb99-de9d653b90c0","resolution":{"observed_at":"2026-05-21T05:24:39.803226Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Quantifying Language Models’ Sensitivity to Spurious Features in Prompt Design","venue":null,"work_id":"f36c0fa8-36d1-4a9f-9d84-fa5ec3e34250","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:19548d7135b0889ec70960efebffd1dcc0f860cda6d37f816367666802a5db98","observation_id":"85711ad8-57a5-46cd-9cbf-518d55031ab4","resolution":{"observed_at":"2026-05-21T05:24:39.786208Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"State of What Art? A Call for Multi-Prompt LLM Eval- uation","venue":null,"work_id":"8d0e54cd-6db4-42e3-b144-e1dd85053115","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:89da256be5dac0cbe95fcd3fb5920ae393c796e2f2573e5c28905ba9b2754934","observation_id":"7150fcf3-12ea-49ad-a8da-cc54ec50bc89","resolution":{"observed_at":"2026-05-21T05:24:39.788401Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices","venue":null,"work_id":"aa1acbec-31d7-4692-b8b3-750d8e5f1559","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:4ca6d20bf455a529ca384d21275e2238938119347ecf11264b0875e9c5d138ca","observation_id":"1fdeccb2-3166-49e3-89db-d06c18bb22ec","resolution":{"observed_at":"2026-05-21T05:24:39.792783Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Data Contamination: From Memorization to Exploitation","venue":null,"work_id":"aaea249e-441e-427c-8ba2-925327233b5d","year":2022},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:01e00ad5c0302440d90bc20dc2824f143a8748cac1285625e033afe17f16eca0","observation_id":"3bf7f789-910d-43e1-8487-79dc26a38268","resolution":{"observed_at":"2026-05-21T05:24:39.799033Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Proving Test Set Contamination in Black Box Language Models","venue":null,"work_id":"88bb0013-50e2-40e8-b9db-584f9dbbc72f","year":2024},"citing_paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-05-21T05:20:01.498711Z"},"links":{"citing_paper":"/paper/2605.21404"},"observation_digest":"sha256:68e5c2236808626b29881f9a58e54a2899c155a25af76193cea6dea9e40060df","observation_id":"0314c8b7-13c6-4e29-b33b-d78a70855a45","resolution":{"observed_at":"2026-05-21T05:24:39.801147Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2605.21404","last_updated":"2026-05-20T17:02:36Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-16T08:00:17.882212Z","submitted_at":"2026-05-20T17:02:36Z","title":"What Twelve LLM Agent Benchmark Papers Disclose About Themselves: A Pilot Audit and an Open Scoring Schema"},"reference_resolution":{"displayed":23,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":3,"verified_fuzzy":20},"total_outbound_references":23},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"thesis":"As of 22 August 2026, this Paper Citation Record lists 23 of 23 outbound references and 1 inbound Pith citation observation for arXiv:2605.21404."}