{"as_of":"2026-08-20T01:21:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d3668739e0be9b26cfb796d63e857033daaa89fb9de3f7999baee2cf689508a8","coverage":[{"denominator":0,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":9,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":9,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":9,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":9,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T15:54:32.977436Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-01T21:16:13.549729Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-12T15:54:32.977436Z","title":"What will it take to fix be nchmarking in natural language understanding? arXiv preprint arXiv:2104.02145 , 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13808","last_updated":"2024-11-21T03:14:31Z","snapshot_observed_at":"2026-08-18T22:43:29.446663Z","submitted_at":"2024-11-21T03:14:31Z","title":"GPAI Evaluations Standards Taskforce: Towards Effective AI Governance","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T15:54:32.977436Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2411.13808"},"observation_digest":"sha256:5244502cf6d66c14b27b2e9bfe7753556f3bff27b68574a60cc23d7e51dcbb65","observation_id":"114205d4-b6f3-48d4-b9a4-18830077255b","resolution":{"observed_at":"2026-08-12T15:54:32.977436Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-10T23:08:44.109938Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.21052","last_updated":"2024-12-30T16:09:33Z","snapshot_observed_at":"2026-08-18T12:43:29.032686Z","submitted_at":"2024-12-30T16:09:33Z","title":"Towards Effective Discrimination Testing for Generative AI","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-10T23:08:44.109938Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2412.21052"},"observation_digest":"sha256:9a7299b188731514687e0f067cc3067694cde4087dd692e887aca2a34028e5c6","observation_id":"c93d6f05-293f-4694-83ec-c08040ca669e","resolution":{"observed_at":"2026-08-10T23:08:44.109938Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-09T04:45:34.570985Z","title":"What will it take to fix benchmarking in natural language understanding?","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.03461","last_updated":"2025-02-05T18:58:19Z","snapshot_observed_at":"2026-08-18T09:40:39.257250Z","submitted_at":"2025-02-05T18:58:19Z","title":"Do Large Language Model Benchmarks Test Reliability?","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-09T04:45:34.570985Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2502.03461"},"observation_digest":"sha256:9a9f245ff9fcd9794bcc2b41891f6de270702719385f45737213b6e71dbbe2d5","observation_id":"5f633ecc-145a-4fa1-99df-696893cb17e7","resolution":{"observed_at":"2026-08-09T04:45:34.570985Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-07T15:44:29.744748Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.14015","last_updated":"2025-06-19T10:05:42Z","snapshot_observed_at":"2026-08-14T13:58:44.564276Z","submitted_at":"2025-05-20T07:09:13Z","title":"AUTOLAW: Enhancing Legal Compliance in Large Language Models via Case Law Generation and Jury-Inspired Deliberation","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T15:44:29.744748Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2505.14015"},"observation_digest":"sha256:18bed63c4786682b3f5d1ad036320062cf3bfa5378e8b2316b067069f9a55dcb","observation_id":"5161b5c4-8a12-4a33-a9a2-18c05efa0da6","resolution":{"observed_at":"2026-08-07T15:44:29.744748Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-06T22:30:28.809315Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.21521","last_updated":"2025-06-29T18:12:45Z","snapshot_observed_at":"2026-08-16T00:40:47.407905Z","submitted_at":"2025-06-26T17:41:35Z","title":"Potemkin Understanding in Large Language Models","version":2},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-06T22:30:28.809315Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2506.21521"},"observation_digest":"sha256:b85b97ec1ce64420d7785b35371551f6aba4a8ee4fe2cb358079271c08c36a52","observation_id":"2aadb985-3932-4443-9b5c-20ebf50d25b2","resolution":{"observed_at":"2026-08-06T22:30:28.809315Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-05T15:43:58.776776Z","title":"What will it take to fix benchmarking in natural language understanding?arXiv preprint arXiv:2104.02145, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.00085","last_updated":"2025-08-27T07:08:44Z","snapshot_observed_at":"2026-08-13T09:09:43.124306Z","submitted_at":"2025-08-27T07:08:44Z","title":"Private, Verifiable, and Auditable AI Systems","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-05T15:43:58.776776Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2509.00085"},"observation_digest":"sha256:00308d0af46160ef356cc5c6b1d4734b6bfb7d9926d0f3b24d570f7acee98458","observation_id":"d7b0d58c-24b5-4c16-808e-07ea8fb92706","resolution":{"observed_at":"2026-08-05T15:43:58.776776Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":"2104.02145","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-07-01T21:16:13.549729Z","title":"Bowman and George E","venue":null,"work_id":"9452a527-b5b7-4145-91e8-49e2f6ee62c2","year":2021},"citing_paper":{"arxiv_id":"2605.12673","last_updated":"2026-05-12T19:22:45Z","snapshot_observed_at":"2026-08-15T14:26:18.460325Z","submitted_at":"2026-05-12T19:22:45Z","title":"Do Androids Dream of Breaking the Game? Systematically Auditing AI Agent Benchmarks with BenchJack","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-05-14T20:31:50.043920Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2605.12673"},"observation_digest":"sha256:d3e929fb75f979e8683d883851031c3f52dfaa45c9bd17aa40cd6797252f8ac5","observation_id":"8741e97a-d21d-4b4d-8341-40c8bcb021d2","resolution":{"observed_at":"2026-05-14T20:32:56.901133Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":"2104.02145","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-07-01T21:16:13.549729Z","title":"Bowman and George E","venue":null,"work_id":"9452a527-b5b7-4145-91e8-49e2f6ee62c2","year":2021},"citing_paper":{"arxiv_id":"2606.01189","last_updated":"2026-05-31T12:11:47Z","snapshot_observed_at":"2026-08-18T05:04:51.949794Z","submitted_at":"2026-05-31T12:11:47Z","title":"The Case for Model Science: Verify, Explore, Steer, Refine","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-06-28T17:24:32.311565Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2606.01189"},"observation_digest":"sha256:4760b08c7e4619c2ae9259e4d91b9bffcde3c6dd6e2cf0d3b0c26afbf7b42318","observation_id":"4dd1879b-a8f5-47f6-8591-e7ad02767f45","resolution":{"observed_at":"2026-07-01T21:16:13.551591Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.02145","snapshot_observed_at":"2026-08-11T10:20:58.755608Z","title":"Bowman and George E","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.09819","last_updated":"2026-08-10T16:39:55Z","snapshot_observed_at":"2026-08-18T21:05:33.598724Z","submitted_at":"2026-08-10T16:39:55Z","title":"Macaron-V1: Towards Open Continual Learning with Self-Improvement and Mixture-of-LoRA","version":1},"reference_index":2026,"source":"pdf_text","source_observed_at":"2026-08-11T10:20:58.755608Z"},"links":{"cited_paper":"/paper/2104.02145","citing_paper":"/paper/2608.09819"},"observation_digest":"sha256:efefce42cda6312f078747a31e30eea7b3bb4f5f84fe6cb7c8228fa008194312","observation_id":"278c8f23-f6ca-4367-9807-70f3fc487741","resolution":{"observed_at":"2026-08-11T10:20:58.755608Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2104.02145/citation-record","integrity":"/paper/2104.02145/integrity","json":"/paper/2104.02145/citation-record.json","paper":"/paper/2104.02145"},"outbound":[],"paper":{"arxiv_id":"2104.02145","last_updated":"2021-10-15T19:10:43Z","latest_version":3,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-16T18:33:44.887657Z","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?"},"reference_resolution":{"displayed":0,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":0},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 20 August 2026, this Paper Citation Record lists 0 of 0 outbound references and 9 inbound Pith citation observations for arXiv:2104.02145."}