{"as_of":"2026-08-08T19:46:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d0aace4a4c1b5db48413d9c925d2b12d2bd050f179da1b8fefa78ac03d33398b","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T04:17:31.943657Z","state":"measured"},{"denominator":27,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":27,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02996/citation-record","integrity":"/paper/2608.02996/integrity","json":"/paper/2608.02996/citation-record.json","paper":"/paper/2608.02996"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.19457","last_updated":"2026-02-14T11:42:30Z","snapshot_observed_at":"2026-08-06T03:52:41.836315Z","submitted_at":"2025-07-25T17:42:32Z","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.19457","snapshot_observed_at":"2026-08-08T04:17:31.686300Z","title":"Agrawal et al","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.686300Z"},"links":{"cited_paper":"/paper/2507.19457","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:830f651219ce6d5d851ee0cbf6e9eddf6ff47ec83a280099b32a21f27110ca84","observation_id":"e3030bc4-0f75-445e-973a-35afed1ec6df","resolution":{"observed_at":"2026-08-08T04:17:31.686300Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.854116Z","title":"Artificial intelligence governance in health systems: Systematic review of frameworks and integrative model proposal.J","venue":null,"work_id":"103babcc-fe53-4994-b276-8aaa0c68be5f","year":2026},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.697353Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:fc70b4fa18759ef76891a52d396109276e39211f8d98d664b19fab294483c674","observation_id":"4674ccfd-5493-4e6f-9107-4215eb480aa9","resolution":{"observed_at":"2026-08-08T04:17:32.861169Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.823060Z","title":"Bender, Timnit Gebru, Angelina McMillan-Major, and Shmargi Shmitchell","venue":null,"work_id":"ed53a2fd-f9bd-48ce-808b-c0b93a1a5957","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.708073Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:1134ec9bde86635fcde639fd8bdbe73c0eba45080bfdb0dc32aff4728f17657a","observation_id":"09513cd2-de6c-49ec-82b0-135719d40440","resolution":{"observed_at":"2026-08-08T04:17:32.830022Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07258","last_updated":"2022-07-12T23:45:14Z","snapshot_observed_at":"2026-08-02T09:20:40.804790Z","submitted_at":"2021-08-16T17:50:08Z","title":"On the Opportunities and Risks of Foundation Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07258","snapshot_observed_at":"2026-08-08T04:17:31.716370Z","title":"On the opportunities and risks of foundation models.arXiv preprint arXiv:2108.07258, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.716370Z"},"links":{"cited_paper":"/paper/2108.07258","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:00af93c8be29753b6c5e1bafe5fbc114c44f299d3fce8380b385731f67d3beb4","observation_id":"8ebf4aa4-c6dd-4a47-9bae-b0e95439821d","resolution":{"observed_at":"2026-08-08T04:17:31.716370Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.789472Z","title":"Sustainability and participation in the digital commons.Interactions, 2017","venue":null,"work_id":"009bb7c5-f72f-497f-9d2b-74e909f6efbc","year":2017},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.726929Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:4aad8bd0e677bad6243bb8113c27c344ecdedaa8666d2a596da6f8c5d6a4974a","observation_id":"1e51c863-8253-40e5-bb5f-4883145be71b","resolution":{"observed_at":"2026-08-08T04:17:32.800347Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1511.04970","last_updated":"2017-02-06T00:51:34Z","snapshot_observed_at":"2026-07-06T04:36:38.735473Z","submitted_at":"2015-11-16T14:29:38Z","title":"Learning about Spanish dialects through Twitter","version":2},"cited_work":{"arxiv_id":"1511.04970","doi":null,"metadata_source":"pith","pith_arxiv_id":"1511.04970","snapshot_observed_at":"2026-08-08T04:17:32.291501Z","title":"Learning about Spanish dialects through Twitter","venue":"stat.ML","work_id":"992d58b4-b955-4e6c-9e2c-473ea8295c4c","year":2015},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.734956Z"},"links":{"cited_paper":"/paper/1511.04970","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:65a75f26ee30a0cac941b71c5d64fa9a083e611227d5649bc8f259c1d495c43c","observation_id":"642d7662-9159-467b-bed3-0b8781df20e7","resolution":{"observed_at":"2026-08-08T04:17:32.300809Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.754136Z","title":"Evaluation gaps in machine learning practice","venue":null,"work_id":"a7b3d0a3-e2ff-4f7c-838f-92d834df9a69","year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.744217Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:38d7c9e303e8dc968353420cbddacd2f8f7c73f64bb19b43dff6c67e3002c9c3","observation_id":"0eeaf8ff-291c-43df-8f9a-739a0b6774a0","resolution":{"observed_at":"2026-08-08T04:17:32.774089Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.727358Z","title":"Epistemic injustice in generative ai","venue":null,"work_id":"8b4d7615-8f04-455f-91f0-2f9f17923be4","year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.752593Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:59817fca83662d821977642c09db111846e9cc52cd68f6636aae595d741a4eb6","observation_id":"fc954717-f020-474a-8ae5-f022f5587956","resolution":{"observed_at":"2026-08-08T04:17:32.734450Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03714","last_updated":"2023-10-05T17:37:25Z","snapshot_observed_at":"2026-08-04T05:21:05.846165Z","submitted_at":"2023-10-05T17:37:25Z","title":"DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03714","snapshot_observed_at":"2026-08-08T04:17:31.763497Z","title":"Dspy: Compiling declarative language model calls into self-improving pipelines.arXiv preprint arXiv:2310.03714, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.763497Z"},"links":{"cited_paper":"/paper/2310.03714","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:7a1b216d757b7462f3d754faa3b3a7f58f7486fd4f6864c5118f9276d73ae3ac","observation_id":"749db3e3-648b-4a62-b588-c4b172907feb","resolution":{"observed_at":"2026-08-08T04:17:31.763497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.701813Z","title":"Latamgpt: Open llms for latin american spanish.huggingface.co/ latam-gpt, 2025","venue":null,"work_id":"5760f6e7-6358-4c50-8e6a-43e553c50e62","year":2025},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.774100Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:4759fe9106aa23b6ca6588260542ea67b8a7046cdc476ce47fb89ef3d270e95a","observation_id":"e0259c74-9d9f-4482-87e0-d7d2c22697a1","resolution":{"observed_at":"2026-08-08T04:17:32.709645Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.670239Z","title":"Some simple economics of open source.The Journal of Industrial Economics, 50(2):197–234, 2002","venue":null,"work_id":"5d4d27ec-a6db-475b-b62c-42966f350134","year":2002},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.785001Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:91693a220f86be366d4caac52a6f48f8ed951b716a5af06e65badbae9e60d114","observation_id":"53d18846-a795-4a20-b919-34433a02ff3b","resolution":{"observed_at":"2026-08-08T04:17:32.676436Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-01T19:14:56.803459Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T04:17:31.793770Z","title":"Holistic evaluation of language models.arXiv preprint arXiv:2211.09110, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.793770Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:22ee07fae4708afe0a70f99001c1c1a59d2ebf03855fa4d1af94b23f25489625","observation_id":"b4feaab7-b720-46eb-8c3f-60db73e005aa","resolution":{"observed_at":"2026-08-08T04:17:31.793770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.640214Z","title":"The medical algorithmic audit.The Lancet Digital Health, 4(3):e152–e163, 2022","venue":null,"work_id":"1a536b2a-612f-4fb3-b451-159e7eebc606","year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.801431Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:63368cf0d1ac2d230bb8a804f3a4bdea407b2c0b26a72b9fd7b08fef657c8186","observation_id":"c6aec451-e7ac-4ae2-9d1d-7639278a9c92","resolution":{"observed_at":"2026-08-08T04:17:32.649175Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.02680","last_updated":"2024-10-05T18:20:58Z","snapshot_observed_at":"2026-07-06T17:25:14.115360Z","submitted_at":"2024-02-05T02:32:09Z","title":"Large Language Models are Geographically Biased","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.02680","snapshot_observed_at":"2026-08-08T04:17:31.810090Z","title":"Lobell, and Stefano Ermon","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.810090Z"},"links":{"cited_paper":"/paper/2402.02680","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:6cfa42c5e95b125f51b2c4a0740adce1c265664132fcd2cbdf8bd5b3fc3b52c6","observation_id":"42b63746-05fd-4fbb-be3d-fb1512eee4ae","resolution":{"observed_at":"2026-08-08T04:17:31.810090Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.614183Z","title":"Robertson","venue":null,"work_id":"2f0abd1a-ced8-40f9-8536-ba6a9973e0fe","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.826397Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:21b55678549bcb376fa55e852c785807c42e60aacd4036d3ed5618158b3b0945","observation_id":"95b7657d-2e2e-4fff-b57a-b316fd25811b","resolution":{"observed_at":"2026-08-08T04:17:32.622604Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.585755Z","title":"Auditing large language models: a three-layered approach.AI and Ethics, 2023","venue":null,"work_id":"d0013735-aa13-45df-8140-a7d9fa14bbea","year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.833689Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:7216352b155ef15d35981f18976a4b10023a8c1b1f5397ed326998640f91572e","observation_id":"fb26bcfd-4787-4e08-a0aa-1068a7ebbb5b","resolution":{"observed_at":"2026-08-08T04:17:32.595137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11695","last_updated":"2024-10-06T17:34:26Z","snapshot_observed_at":"2026-07-06T18:32:19.794373Z","submitted_at":"2024-06-17T16:12:03Z","title":"Optimizing Instructions and Demonstrations for Multi-Stage Language Model Programs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11695","snapshot_observed_at":"2026-08-08T04:17:31.840402Z","title":"Optimizing instructions and demonstrations for multi-stage language model programs","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.840402Z"},"links":{"cited_paper":"/paper/2406.11695","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:dd828443a35c927c37b6d87484c1f13afd8bf2f89cd88e346a31184acfd403e8","observation_id":"e4a1719a-3cac-4952-b079-947b6add83a4","resolution":{"observed_at":"2026-08-08T04:17:31.840402Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2003.12206","last_updated":"2020-12-30T21:32:34Z","snapshot_observed_at":"2026-08-08T02:21:51.646976Z","submitted_at":"2020-03-27T02:16:25Z","title":"Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2003.12206","snapshot_observed_at":"2026-08-08T04:17:31.852636Z","title":"Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program).JMLR, 2020","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.852636Z"},"links":{"cited_paper":"/paper/2003.12206","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:0ba220be636a4e0799b2d42b4a65440a0677759bcbe08d02cd4091c875a8b5be","observation_id":"19c159ac-e7ef-4e43-849b-796c82b9d614","resolution":{"observed_at":"2026-08-08T04:17:31.852636Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.555452Z","title":"A governance model for the application of ai in health care.J","venue":null,"work_id":"0eb6ee66-cb88-487c-a379-a83e7a684daa","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.870231Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:dfde0674ae08816516c3bafd872c381559c0ce750785af17a29587d4a34f46f6","observation_id":"db40ef7b-2e12-48af-92ea-ef1df90eab98","resolution":{"observed_at":"2026-08-08T04:17:32.563111Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.12990","last_updated":"2024-11-20T02:38:24Z","snapshot_observed_at":"2026-07-06T19:52:55.369337Z","submitted_at":"2024-11-20T02:38:24Z","title":"BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.12990","snapshot_observed_at":"2026-08-08T04:17:31.890320Z","title":"Betterbench: Assessing ai benchmarks, uncovering issues, and establishing best practices.arXiv preprint arXiv:2411.12990, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.890320Z"},"links":{"cited_paper":"/paper/2411.12990","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:63f690801f81e416c67db8eaac78f69cecc2eb13e6aa4411a3387b944236a211","observation_id":"30a49794-cdea-44d1-9de1-eee8fddb6218","resolution":{"observed_at":"2026-08-08T04:17:31.890320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.518991Z","title":"Prompt programming for large language models: Beyond the few-shot paradigm","venue":null,"work_id":"49830bd2-25b0-4130-a8f2-4bb93e2209c8","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.896304Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:762f21e5dc93a2341b8be9b63e03a4faa145bab9c3feb45daa299f81cbe14fae","observation_id":"cbcbb797-b100-4c59-9a25-985ce7039c67","resolution":{"observed_at":"2026-08-08T04:17:32.528048Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.486494Z","title":"Digital sovereignty and artificial intelligence: a normative approach.AI and Ethics, 2024","venue":null,"work_id":"b2f1234f-01f7-4f61-8472-94a726781d82","year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.906077Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:235b0de9fc7de594455446540143cc0244b5d5fb113b2f29c31b76464b0604a8","observation_id":"e66aa136-350d-47d2-8231-2b6f8c5697fe","resolution":{"observed_at":"2026-08-08T04:17:32.496417Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.450786Z","title":null,"venue":null,"work_id":"9b0ef418-93da-4079-9180-50ddd438921f","year":2020},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.912582Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:a27667d78d4c0c87233ebef95da62c3154b3c69e73619882636d2c0f3b1ae56d","observation_id":"04d392bc-fb5d-45ff-b7ff-db25917f4cbe","resolution":{"observed_at":"2026-08-08T04:17:32.461929Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.419351Z","title":"Ethics of ai and cybersecurity when sovereignty is at stake.Minds and Machines, 2019","venue":null,"work_id":"ca5bfc68-2db3-49c9-afc2-f08538935706","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.920536Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:ccf0314bff17f66746bac756ae8629e6ca49ae8e4230a4f50a58dd64b30c2484","observation_id":"8aa9fb68-3c7e-4135-989d-288471c6ad87","resolution":{"observed_at":"2026-08-08T04:17:32.427949Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.389954Z","title":"Weber et al","venue":null,"work_id":"40b8b7f1-352b-49cf-8926-ec2d6757e2c6","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.929240Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:89274aab2388c80aaa70217278b65d740b30452f9a9096be8b83c2ebdf765b37","observation_id":"4ede66d8-890b-45d7-92ef-18c19eb412bd","resolution":{"observed_at":"2026-08-08T04:17:32.402349Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04244","last_updated":"2024-06-06T16:41:39Z","snapshot_observed_at":"2026-07-30T15:43:06.151242Z","submitted_at":"2024-06-06T16:41:39Z","title":"Benchmark Data Contamination of Large Language Models: A Survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04244","snapshot_observed_at":"2026-08-08T04:17:31.936704Z","title":"Benchmark data contamination of large language models: A survey.arXiv preprint arXiv:2406.04244, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.936704Z"},"links":{"cited_paper":"/paper/2406.04244","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:2637108677a9e0f7c6fc6b94bdf8fdcca469c7375712f145d031000bbd47b9c8","observation_id":"f7059a58-e641-411b-a13a-f79dd2a3bfea","resolution":{"observed_at":"2026-08-08T04:17:31.936704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-08T04:17:31.943657Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.943657Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:56424452e0ef3ab77e14e149fc6b062831a5acc3368bb2ff95a31c063cabbd15","observation_id":"81c84ca6-74dd-4929-91d6-7f98a1b65a2f","resolution":{"observed_at":"2026-08-08T04:17:31.943657Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-08T04:12:59.819038Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":11,"verified_exact":1,"verified_fuzzy":15},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 0 inbound Pith citation observations for arXiv:2608.02996."}