{"as_of":"2026-08-07T22:37:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b2ff618bb2d16e2337c814c810b5f5a5aa103afa9b9232ed06867ae289a25d35","coverage":[{"denominator":94,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":94,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T13:38:01.264257Z","state":"measured"},{"denominator":94,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":94,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2507.20419/citation-record","integrity":"/paper/2507.20419/integrity","json":"/paper/2507.20419/citation-record.json","paper":"/paper/2507.20419"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:51.994781Z","title":"Principles of Evaluation in Natural Language Processing,","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:51.994781Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5cc6030c97351b60a256221ddd000d6ecb79bbcf00f7bd22380db3fdf2207b7c","observation_id":"689c45d2-87cc-4baa-a256-e26be0f53ed6","resolution":{"observed_at":"2026-08-06T13:37:51.994781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.365652Z","title":"Natural Language Inference ,","venue":null,"work_id":null,"year":1973},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.365652Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:499a6ccbe57484285a4bd00298748b6c8f74303ed2c99f94e77ae7b4cd05f820","observation_id":"7680c4c5-ab05-40de-bfb9-5b3bdd50f0df","resolution":{"observed_at":"2026-08-06T13:37:52.365652Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.487079Z","title":"PROBABILISTIC TEXTUAL ENTAILMENT: GENERIC APPLIED MODELING OF LANGUAGE VARIABILITY,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.487079Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:44681907f5702408ea2b031a6b2b4ded7b1cf8a4bdb778844d87639ca4d61814","observation_id":"1259cf48-029a-46e1-a7d5-840d469b0b86","resolution":{"observed_at":"2026-08-06T13:37:52.487079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.725915Z","title":"Probabilistic textual entailment: Generic applied modeling of language variability,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.725915Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c94ac11d771388c6697718ccf90d5f3f9a09864e82d1e97c6d50b08070b0742a","observation_id":"927f39e9-f3cc-454e-bf7d-d4a33ded723b","resolution":{"observed_at":"2026-08-06T13:37:52.725915Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.872893Z","title":"The Seventh PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.872893Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7de8ce6bfab0bbfeafccd1d1f89df1ac2ac4fc1310e0da592d8e99af2f2f4522","observation_id":"f96c17a0-920b-4d4d-8f46-579190916b18","resolution":{"observed_at":"2026-08-06T13:37:52.872893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.153624Z","title":"The Sixth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.153624Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:9dbdbd7d961a8387cf0883988900f80e9a1fbac7c21c15997fc10b99c2df4983","observation_id":"55544bb1-66cf-4be4-b593-6d34d9c0c05d","resolution":{"observed_at":"2026-08-06T13:37:53.153624Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.279800Z","title":"The Fifth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.279800Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3bcfe902594513eca329867ca3e200a2b1bd40e815b8e375457eecb3338311f2","observation_id":"7fe02df0-38d3-44dc-8c54-1e59a0b1d7b2","resolution":{"observed_at":"2026-08-06T13:37:53.279800Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.425491Z","title":"The Third PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.425491Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d2a3c9f51f026cdd999f4e0d50841c669aa0519cf4e7a7b42d624ca67d2a2c0e","observation_id":"9ece044d-c086-4df3-bf6b-0e44cb3cd691","resolution":{"observed_at":"2026-08-06T13:37:53.425491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.601192Z","title":"The Second PASCAL Recognising Textual Entailment Challenge,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.601192Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:27cbcd0a5fde3d8c9601f1ca37206036c79ecd5d821a1089994ba8b22759a2c6","observation_id":"92ba844e-0949-47ea-addb-9ab860166dde","resolution":{"observed_at":"2026-08-06T13:37:53.601192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.769882Z","title":"The PASCAL Recognising Textual Entailment Challenge,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.769882Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:a63a22b6e1a9e3dbcdc43a7eb8ad50cfe8f348a3b680d3ddc31067bbfb86a1ee","observation_id":"1b9991c9-4ae9-4ed4-88b2-123b54dcb7e0","resolution":{"observed_at":"2026-08-06T13:37:53.769882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.910758Z","title":"The Fourth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.910758Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:8345d3b0aae8831a0ddf206b7b269e33e32aaa8fe7e2b5090856ded2e8891abb","observation_id":"cae6e79b-fc41-44fd-9fbe-c77460cc6ebf","resolution":{"observed_at":"2026-08-06T13:37:53.910758Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.2200/s00509ed1v01y201305hlt023","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:04.680528Z","title":"Recognizing Textual Entailment: Models and Applications,","venue":null,"work_id":"741721d2-27d1-4c6d-98ce-e16bf33b7631","year":2013},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.002707Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2ec9f411d753c845c9f1e9927eeccbcab3871382929722509144df13a0b4ff86","observation_id":"f78a0d00-f5f3-4b3f-b020-b63fb381a23b","resolution":{"observed_at":"2026-08-06T13:38:04.882356Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.120712Z","title":"The Winograd Schema Challenge,","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.120712Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e605ad128786935c0c65534cbdc9739318d360cff7ef14aaf48d6fef535353d0","observation_id":"ebe93d99-08dc-4bbf-b71a-f2a226fd7535","resolution":{"observed_at":"2026-08-06T13:37:54.120712Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.187851Z","title":"A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.187851Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:33f678f0c9d21465424a12b9c6ca23bf7685eec6f28c74a9ace55c633f739ee0","observation_id":"08857552-76f4-423c-8d58-10b41f0d7ba5","resolution":{"observed_at":"2026-08-06T13:37:54.187851Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.254334Z","title":"A large annotated corpus for learning natural language inference,","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.254334Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1e11e6ba212b8f43e9cd0188ca1255a8b2e5923be9bc541479d8d880098259b1","observation_id":"66995a61-e64d-4e3c-9ff5-05e584fbf5a5","resolution":{"observed_at":"2026-08-06T13:37:54.254334Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.336819Z","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.336819Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ca364f9605195513df9e7987a92b874de7fbfbe8fba807103a89b6f0e9809616","observation_id":"3597f193-4c39-413e-ad2a-facdc16169b5","resolution":{"observed_at":"2026-08-06T13:37:54.336819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.02311","last_updated":"2022-10-05T06:02:24Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-05T16:11:45Z","title":"PaLM: Scaling Language Modeling with Pathways","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.02311","snapshot_observed_at":"2026-08-06T13:37:54.375277Z","title":"PaLM: Scaling Language Modeling with Pathways,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.375277Z"},"links":{"cited_paper":"/paper/2204.02311","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:bad67230f80ea7e10a4fe9e92d9d18959f0312aec74de95ae096644bd1a7f7a7","observation_id":"f1d0f05a-eadd-4e8d-b8ea-4d6e4d976f47","resolution":{"observed_at":"2026-08-06T13:37:54.375277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.01853","last_updated":"2022-12-04T15:36:18Z","snapshot_observed_at":"2026-07-06T14:26:36.704762Z","submitted_at":"2022-12-04T15:36:18Z","title":"Toward Efficient Language Model Pretraining and Downstream Adaptation via Self-Evolution: A Case Study on SuperGLUE","version":1},"cited_work":{"arxiv_id":"2212.01853","doi":null,"metadata_source":"pith","pith_arxiv_id":"2212.01853","snapshot_observed_at":"2026-08-06T13:38:06.324880Z","title":"Toward Efficient Language Model Pretraining and Downstream Adaptation via Self-Evolution: A Case Study on SuperGLUE","venue":"cs.CL","work_id":"15650321-4431-4899-8f85-54caa9bfb14a","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.473408Z"},"links":{"cited_paper":"/paper/2212.01853","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:702b595148f35fca82f8f179b9142d8dba924b8efdc3fc6ea1a7f0662c77e2b6","observation_id":"77c1d34e-f5b0-49df-a804-4d570799d029","resolution":{"observed_at":"2026-08-06T13:38:06.437083Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.812775Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach,","venue":null,"work_id":"7665aec6-b162-44d1-ad55-165aacd9d81c","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.552526Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fddddeefa0541291737fa0f7bda9994f4cf8068b7c4b6c9a26cd5b03d543b197","observation_id":"3368c16d-2b2d-4928-be27-0ec4dcc1c3e8","resolution":{"observed_at":"2026-08-06T13:38:07.816504Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.799883Z","title":"Semantics-aware BERT for Language Understanding,","venue":null,"work_id":"33c2c9d3-ddb9-4404-b6b1-9d4dc3904614","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.618700Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:61e1ab8904388d4759970e845e6ca96f487b26de06186f61955230d3b1881c44","observation_id":"bb834d31-58b1-488c-a015-781ebdc0d131","resolution":{"observed_at":"2026-08-06T13:38:07.803775Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.787239Z","title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding,","venue":null,"work_id":"4c70d31f-f2e5-44d5-b88d-6d7d1fb808c5","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.677424Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1d137ac1b87cffa84aeff4b54b45b1f7bc1c05dc4f179e326d899c609b912ee7","observation_id":"6d1c5b9b-b0dd-4cf2-8f26-47ae3d0b1789","resolution":{"observed_at":"2026-08-06T13:38:07.791260Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2208.01448","last_updated":"2022-08-03T12:42:52Z","snapshot_observed_at":"2026-08-07T10:18:54.878316Z","submitted_at":"2022-08-02T13:30:07Z","title":"AlexaTM 20B: Few-Shot Learning Using a Large-Scale Multilingual Seq2Seq Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.01448","snapshot_observed_at":"2026-08-06T13:37:54.767217Z","title":"AlexaTM 20B: Few-Shot Learning Using a Large-Scale Multilingual Seq2Seq Model,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.767217Z"},"links":{"cited_paper":"/paper/2208.01448","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:8f7229fea030db1604a02487c52abe83beb5b57cac25d794a37a03d2a35fcdf4","observation_id":"92347395-5567-476b-b319-f3dd66da57aa","resolution":{"observed_at":"2026-08-06T13:37:54.767217Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.17564","last_updated":"2023-12-21T06:21:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-03-30T17:30:36Z","title":"BloombergGPT: A Large Language Model for Finance","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.17564","snapshot_observed_at":"2026-08-06T13:37:54.862401Z","title":"BloombergGPT: A Large Language Model for Finance,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.862401Z"},"links":{"cited_paper":"/paper/2303.17564","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:8279cff7651f291ea3a36b275ed2870dc8a1df54c5a33ccac2e2ffab3f8c6f02","observation_id":"c865e660-91f3-44ca-8312-1150c6491cd4","resolution":{"observed_at":"2026-08-06T13:37:54.862401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.09263","last_updated":"2024-12-13T06:28:11Z","snapshot_observed_at":"2026-07-06T20:05:55.982214Z","submitted_at":"2024-12-12T13:21:09Z","title":"First Train to Generate, then Generate to Train: UnitedSynT5 for Few-Shot NLI","version":2},"cited_work":{"arxiv_id":"2412.09263","doi":"10.48550/arxiv.2412.09263","metadata_source":"pith","pith_arxiv_id":"2412.09263","snapshot_observed_at":"2026-08-06T18:16:30.548575Z","title":"First Train to Generate, then Generate to Train: UnitedSynT5 for Few-Shot NLI","venue":"cs.CL","work_id":"8101b417-2b82-44eb-a441-b4feef34ab86","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.934359Z"},"links":{"cited_paper":"/paper/2412.09263","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:4e3a9ca59c7a0e212cf41a09d39212cab025c30b4cb2d3b14c0da70bbde459b4","observation_id":"3ff88a52-1719-405a-80ac-5075153207c8","resolution":{"observed_at":"2026-08-06T13:38:04.472952Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.981376Z","title":"ByT5: Towards a Token-Free Future with Pre-trained Byte-to-Byte Models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.981376Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:44089c2534e93ad40d92a5c65338ab1698bd4232598ca946fa06ff84b2e88038","observation_id":"a6ec2638-a7ae-49e6-968c-5deaf8a1b8fe","resolution":{"observed_at":"2026-08-06T13:37:54.981376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2010.12821","last_updated":"2020-10-24T07:43:00Z","snapshot_observed_at":"2026-07-06T10:08:08.004416Z","submitted_at":"2020-10-24T07:43:00Z","title":"Rethinking embedding coupling in pre-trained language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2010.12821","snapshot_observed_at":"2026-08-06T13:37:55.076184Z","title":"Rethinking embedding coupling in pre-trained language models,","venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.076184Z"},"links":{"cited_paper":"/paper/2010.12821","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7d01c0bb577b762c5068fab52a85e6c01aa74b2c547b29d7e32498497d335309","observation_id":"3fa32c45-f2d4-4d46-9df6-b83f036f226a","resolution":{"observed_at":"2026-08-06T13:37:55.076184Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.158177Z","title":"mGPT: Few-Shot Learners Go Multilingual,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.158177Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:20b34d0e2684a96608e22a9648161af0759317f31d19496ea695f65b44eb3a7c","observation_id":"5b3cc215-56f3-4f27-9ab0-4497e504c3bd","resolution":{"observed_at":"2026-08-06T13:37:55.158177Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.774270Z","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention,","venue":null,"work_id":"01aa2202-8cf2-4dc1-abb8-f83fba8b3ecf","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.202142Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3f59fc503884adbb53baad61b54ec21136d2b0d65c2162222442d2e950965ccf","observation_id":"c363be4e-caae-41c6-acfb-d19496db7f62","resolution":{"observed_at":"2026-08-06T13:38:07.778137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.167222Z","title":"Available: https://aclanthology.org/2007.tal-1.1","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.167222Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2ca689831c8b36fb1bace28d844b19a360c56b065fd14d2e074743d3ae369bd2","observation_id":"c9666e37-80af-4a35-9a5a-f5a4ba0d4a8e","resolution":{"observed_at":"2026-08-06T13:37:52.167222Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.271734Z","title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.271734Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:03c4cab2828032af78ea83c72e6216d9875e83687ee7071dab389c326fa0b73d","observation_id":"5f610143-a1b9-4151-adf0-e3e4041827f1","resolution":{"observed_at":"2026-08-06T13:37:55.271734Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2020.sustainlp-1.17","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.970984Z","title":"SqueezeBERT: What can computer vision teach NLP about efficient neural networks?,","venue":null,"work_id":"6e3c8a62-be6d-422d-9dcd-d2f8a7aaa15a","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.343223Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3d89008ecf1a320b3d5e26b5eb49a32893e0b08bd356ac15798acc343a2f54b7","observation_id":"282848ce-90e3-4530-9594-25e0b0ed6caa","resolution":{"observed_at":"2026-08-06T13:38:04.069737Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1910.01108","last_updated":"2020-03-01T02:57:50Z","snapshot_observed_at":"2026-08-07T19:07:36.327251Z","submitted_at":"2019-10-02T17:56:28Z","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.01108","snapshot_observed_at":"2026-08-06T13:37:55.387682Z","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter,","venue":null,"work_id":null,"year":1910},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.387682Z"},"links":{"cited_paper":"/paper/1910.01108","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fd30d793892a73ba69c34e9bf3fdceb62ffbb5cb22eabf9b9d88aa1221126311","observation_id":"9a9672f9-0a89-4189-b7b4-577db920ad26","resolution":{"observed_at":"2026-08-06T13:37:55.387682Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.761477Z","title":"A Dataset for Arabic Textual Entailment,","venue":null,"work_id":"0a795e51-c17d-4ddb-8eb2-9c0d0abb9036","year":2013},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.479023Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e8561735acfd40b5ae74b9ee984c289fe7a6794e35500b290200451e4405d35e","observation_id":"ee6d3f64-5603-453d-9f90-1a7a75c00cf4","resolution":{"observed_at":"2026-08-06T13:38:07.765397Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.7494/csci.2023.24.2.4378","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.837797Z","title":"ARNLI: ARABIC NATURAL LANGUAGE INFERENCE ENTAILMENT AND CONTRADICTION DETECTION,","venue":null,"work_id":"2c7c0201-c157-4ec0-b506-613dbd14d488","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.613730Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ebe707e2852bc9c2cb42b4beb132e819f894623770a1977347314561d26b758e","observation_id":"3278f0e6-242a-41c4-b4e6-b98a3facfe33","resolution":{"observed_at":"2026-08-06T13:38:03.913885Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s10579-024-09731-1","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.689269Z","title":"ArEntail: manually-curated Arabic natural language inference dataset from news headlines,","venue":null,"work_id":"de78ad1a-5732-4cd3-afcd-5ea9d35ad8bd","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.710306Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1097e6daad2b753d76e5e59eb58c63357254b05130b8c42e20e727a4a062cb2c","observation_id":"bd9dd812-c1de-45af-9eb6-6662e3eb03f9","resolution":{"observed_at":"2026-08-06T13:38:03.723052Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.749025Z","title":"Baselines and Test Data for Cross-Lingual Inference,","venue":null,"work_id":"a3332068-8b96-4621-8d40-12330e681070","year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.827796Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ee82b4c2ed3d8651ed3df6fb8d8e0f7ef040c6132cd967ae9b4bca0a87805a51","observation_id":"1a673ff0-d2b4-49a6-98f6-a8169e2a117e","resolution":{"observed_at":"2026-08-06T13:38:07.752969Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.900030Z","title":"XNLI: Evaluating Cross-lingual Sentence Representations,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.900030Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e57c66121af234cc6e32877c0c31ceb98fb15b7d7e0602e69ffbe8d29bc76cc8","observation_id":"72509850-399e-4755-a1a4-301479c834bc","resolution":{"observed_at":"2026-08-06T13:37:55.900030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.976655Z","title":"Benchmarking Zero-shot Text Classification: Datasets, Evaluation and Entailment Approach,","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.976655Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:be4ec8a72f36e0ffefa03515d9593548b0273f7e670c4f548a6c8f0c6145014f","observation_id":"5eef9a04-89dc-4f4f-915a-623b072993da","resolution":{"observed_at":"2026-08-06T13:37:55.976655Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.072296Z","title":"BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.072296Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d5ff409b3be2983bb70dd35a2c59441608e26363925ea2fcba48db5e0e3d8887","observation_id":"fed28c6d-80b1-4c58-a42d-1a3e195df645","resolution":{"observed_at":"2026-08-06T13:37:56.072296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1907.11692","last_updated":"2019-07-26T17:48:29Z","snapshot_observed_at":"2026-07-31T22:31:37.910868Z","submitted_at":"2019-07-26T17:48:29Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.11692","snapshot_observed_at":"2026-08-06T13:37:56.192293Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach,","venue":null,"work_id":null,"year":1907},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.192293Z"},"links":{"cited_paper":"/paper/1907.11692","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:0bfb578b45a9b4b3da7d0ff3f3852ca5d7294ce5824e1f20c9a23dceff60f29f","observation_id":"134ae3ce-000c-4867-b024-63f1140fcd0f","resolution":{"observed_at":"2026-08-06T13:37:56.192293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2002.10957","last_updated":"2020-04-06T02:53:18Z","snapshot_observed_at":"2026-07-06T09:00:01.184557Z","submitted_at":"2020-02-25T15:21:10Z","title":"MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.10957","snapshot_observed_at":"2026-08-06T13:37:56.242266Z","title":"MiniLM: Deep Self-Attention Distillation for Task- Agnostic Compression of Pre-Trained Transformers,","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.242266Z"},"links":{"cited_paper":"/paper/2002.10957","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:25d1e6cc2fe663319e910518bc33b4df24ec4eea81dea4e64c7d6c089408e419","observation_id":"4a4fdf83-36d2-47a1-9a3a-bf5c34d8e745","resolution":{"observed_at":"2026-08-06T13:37:56.242266Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2111.09543","last_updated":"2023-03-24T09:17:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-11-18T06:48:00Z","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.09543","snapshot_observed_at":"2026-08-06T13:37:56.318397Z","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient- Disentangled Embedding Sharing,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.318397Z"},"links":{"cited_paper":"/paper/2111.09543","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:796205ec3e12c13ce2a3f9155c80d04caded55e54c8f648b77d0d66a5087f307","observation_id":"6e8521c3-0535-431a-adb0-cf7b87e61d38","resolution":{"observed_at":"2026-08-06T13:37:56.318397Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.411095Z","title":"Less annotating, more classifying: Addressing the data scarcity issue of supervised machine learning with deep transfer learning and BERT-NLI,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.411095Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:b39722bca1184610bc061ea1105b51f60bf3741b43a0a763bf8ff5ce2cea8f19","observation_id":"16c9bc1a-6cad-42d4-b394-b7fbd6ae360d","resolution":{"observed_at":"2026-08-06T13:37:56.411095Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.506650Z","title":"GPTAraEval: A Comprehensive Evaluation of ChatGPT on Arabic NLP,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.506650Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e533ad0af71da51df850054ff72e7fa7e9652e2610a05742dd7063b2fc2f0d7a","observation_id":"667d3618-e2f9-4aad-aed6-1437fbae2362","resolution":{"observed_at":"2026-08-06T13:37:56.506650Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.gem-1.6","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.459254Z","title":"CLSE: Corpus of Linguistically Significant Entities,","venue":null,"work_id":"69dc3717-405e-4737-aea9-aec552e180cd","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.576929Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e19d559e3eafffc4ac37f715c6986769b580022e4fafedff2f2c12b19746cff3","observation_id":"40349f99-cd76-426c-8308-65af190c6935","resolution":{"observed_at":"2026-08-06T13:38:03.524186Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.721456Z","title":"The GEM Benchmark: Natural Language Generation, its Evaluation and Metrics,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.721456Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5b6f2e7a1cbd773040156b13f53b7b3ddc9b43dfdbeb59535a293c4240a6b32e","observation_id":"6780fe5d-533e-4a91-b726-fad1dc3b9773","resolution":{"observed_at":"2026-08-06T13:37:56.721456Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.833242Z","title":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.833242Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:a73ae39d2826ee5c930d709e13d15de5f9e10053c09d8ff2ae266cb5f9cb1a42","observation_id":"d9bdb60a-d4bf-4124-a838-90cf873f7543","resolution":{"observed_at":"2026-08-06T13:37:56.833242Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.emnlp-main.360","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.269514Z","title":"IndicNLG Benchmark: Multilingual Datasets for Diverse NLG Tasks in Indic Languages,","venue":null,"work_id":"668088ee-1426-4e06-9747-44a8d37ba8b9","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.920313Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:30b5d9cd46c0a6f7fba593647b93e1b67b3783751c2e6fe3e8639ef1785ab44c","observation_id":"8d70fd07-bf64-4b58-9148-ee1014d64153","resolution":{"observed_at":"2026-08-06T13:38:03.343289Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.001448Z","title":"MTG: A Benchmark Suite for Multilingual Text Generation,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.001448Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:9866225be5111f88b33e2edecfbcfdeb990a8036ad12d949df087d70ccad548e","observation_id":"cefb55ee-4493-4e6d-a886-cfd1123f63fa","resolution":{"observed_at":"2026-08-06T13:37:57.001448Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.121190Z","title":"IndoNLG: Benchmark and Resources for Evaluating Indonesian Natural Language Generation,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.121190Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:41f80e8dcb503d7ecbad850b0b0265e89cecc1c198f190420b30b145ee3ee56c","observation_id":"d9de5286-75d9-4675-8b07-727b8602c4a2","resolution":{"observed_at":"2026-08-06T13:37:57.121190Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.735726Z","title":"Dolphin: A Challenging and Diverse Benchmark for Arabic NLG,","venue":null,"work_id":"6bc2c930-f215-4b49-9331-60957cd943e7","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.234294Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:700024d4836de014b9c72886fea2ad77a3d43b8a96cad6b951fb28f807133891","observation_id":"5bcbe44a-2669-4ef9-a89d-6eb813094b39","resolution":{"observed_at":"2026-08-06T13:38:07.740001Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.722853Z","title":"TURJUMAN: A Public Toolkit for Neural Arabic Machine Translation,","venue":null,"work_id":"067c0c5e-25d1-44f3-b3c4-ce4b1d54aeb9","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.449397Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:16c9d5cf713f7aeb900bde06b09a7143f81506a65ac505c4c774901be38b599e","observation_id":"8720e990-6952-456b-85f3-0f4c21cbe7cb","resolution":{"observed_at":"2026-08-06T13:38:07.727123Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.376864Z","title":"AraBench: Benchmarking Dialectal Arabic-English Machine Translation,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.376864Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:0a463af7a2694b8a90d64c31c026bf9a124e619436d1c7cea3aa06798509b8af","observation_id":"ddab1a97-4f81-4959-a22c-a35ea5dffcae","resolution":{"observed_at":"2026-08-06T13:37:57.376864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.717646Z","title":"BanglaNLG and BanglaT5: Benchmarks and Resources for Evaluating Low-Resource Natural Language Generation in Bangla,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.717646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ea3fbf8f7082826d26072af31b831045d57f82cfcf987f27b17a079cbd6053e6","observation_id":"71743e22-ff20-481c-9453-e97fd137ca06","resolution":{"observed_at":"2026-08-06T13:37:57.717646Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.acl-long.47","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:03.079077Z","title":"AraT5: Text-to-Text Transformers for Arabic Language Generation,","venue":null,"work_id":"3f98c350-2bc3-4cce-a797-5c218556f7ac","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.586733Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:186f451b7cdbce7c011855aa1936e378833226850990ce97fb0a905fe9ead64b","observation_id":"e9f11951-f937-49e1-9aee-a16f37f9f3d7","resolution":{"observed_at":"2026-08-06T13:38:03.156193Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2112.13610","last_updated":"2022-06-14T07:19:35Z","snapshot_observed_at":"2026-08-06T04:10:30.783266Z","submitted_at":"2021-12-27T11:08:58Z","title":"CUGE: A Chinese Language Understanding and Generation Evaluation Benchmark","version":2},"cited_work":{"arxiv_id":"2112.13610","doi":null,"metadata_source":"pith","pith_arxiv_id":"2112.13610","snapshot_observed_at":"2026-08-06T13:38:05.760935Z","title":"CUGE: A Chinese Language Understanding and Generation Evaluation Benchmark","venue":"cs.CL","work_id":"165e321a-4b4d-4ba6-81fe-dfa4b92d8cb8","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.924811Z"},"links":{"cited_paper":"/paper/2112.13610","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:f15a84f75b86a4d3202eefc83197308a931f82fac76202b75fb14b7a56199b30","observation_id":"dfb6a474-923b-401a-b713-bd14cc433a19","resolution":{"observed_at":"2026-08-06T13:38:05.964795Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.emnlp-main.369","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:02.848387Z","title":"PhoMT: A High-Quality and Large-Scale Benchmark Dataset for Vietnamese-English Machine Translation,","venue":null,"work_id":"dffde7be-5c12-4987-945f-0166b91a8fc8","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.828392Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7c31efaf2607a97ab006af75035f52da54a1d0c3cb297282656d9309b58737e7","observation_id":"697a2908-5568-4ec5-aa20-637db9fab701","resolution":{"observed_at":"2026-08-06T13:38:02.957624Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.710413Z","title":"Benchmarking Multidomain English-Indonesian Machine Translation,","venue":null,"work_id":"bdf0d404-1a27-4082-a32b-5a16bc0ea216","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.142558Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:65d2263474e1c154c154ce55a4b9ef40fd15ca63f5fe5cc9ed4d3b9702a627c9","observation_id":"6026dce6-d303-4a06-95d5-120a540ddcfb","resolution":{"observed_at":"2026-08-06T13:38:07.714116Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.998337Z","title":"LOT: A Story-Centric Benchmark for Evaluating Chinese Long Text Understanding and Generation,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.998337Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ed126a3eb07c5d64810333cf8aa9d29bac2a13f35678094207aeb27ebe263e9a","observation_id":"733fc5c9-80fc-4805-99d5-8ee6d3a4d02c","resolution":{"observed_at":"2026-08-06T13:37:57.998337Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.697742Z","title":"XGLUE: A New Benchmark Dataset for Cross-lingual Pre-training, Understanding and Generation,","venue":null,"work_id":"c25d6efb-5c22-459b-9ba4-8ef7c1470418","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.317146Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1346fd0bfd968f82b4ef822b4bc3172b2f3a529ba522567bfe2cccb48db37e6c","observation_id":"ec315073-34a0-4046-908d-7d5c555da901","resolution":{"observed_at":"2026-08-06T13:38:07.701733Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.findings-acl.36","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:02.653006Z","title":"GLGE: A New General Language Generation Evaluation Benchmark,","venue":null,"work_id":"e1696b0e-c849-45c5-87bc-9f1646f44f2d","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.240465Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:be56a3fa7999481d8af4767e3c6d712d9f49077b08ddf5eec37dfed479e7194a","observation_id":"834cf8cc-1dd7-40e3-a06a-18e24224514f","resolution":{"observed_at":"2026-08-06T13:38:02.737908Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.638320Z","title":"XTREME-R: Towards More Challenging and Nuanced Multilingual Evaluation,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.638320Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e662fabd1f6e1a3169b71f3edf82af1215486b026aaf9d63e3ba0d20d87e4230","observation_id":"cd68d2a5-e9de-4fd4-84b3-337170cf595c","resolution":{"observed_at":"2026-08-06T13:37:58.638320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.672356Z","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems,","venue":null,"work_id":"64a60517-0290-4df0-af69-bf187105c3ac","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.735056Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2f134948b1f30d559c96afb8ea8856c3279418f1cde93d4aa4e2fea5424b4502","observation_id":"5d447d3c-7d5e-4330-9a75-7b9517c6d21f","resolution":{"observed_at":"2026-08-06T13:38:07.676377Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.685073Z","title":"XTREME: A Massively Multilingual Multi-task Benchmark for Evaluating Cross-lingual Generalization,","venue":null,"work_id":"78d7922a-f724-4e1f-91a0-2ad93745d08b","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.530760Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:271ff627fd5fd8025f9a9cc6c7513d2c9da6d65f605e59d3ee7507f459135efc","observation_id":"b22bb799-41d6-4867-8b87-9c6c7755d77d","resolution":{"observed_at":"2026-08-06T13:38:07.689029Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.891569Z","title":"ARBERT & MARBERT: Deep Bidirectional Transformers for Arabic,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.891569Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:bf5cc7a77536830932fa97a6217b0ad53e27c7317b3bc1ba6327934f802382b7","observation_id":"91ba2886-2bae-4cb1-b74f-6be8ae3ec06f","resolution":{"observed_at":"2026-08-06T13:37:58.891569Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.647310Z","title":"ORCA: A Challenging Benchmark for Arabic Language Understanding,","venue":null,"work_id":"4101760b-245f-49be-b862-a65faa2428d1","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.994414Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:4ef39e58e09ef4226df00c4c450f1dc161224a890a1e588653293fc52af41c4f","observation_id":"6c24f0c1-3e5d-44a9-bc2a-ea910c5188b6","resolution":{"observed_at":"2026-08-06T13:38:07.651628Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.659997Z","title":"ALUE: Arabic Language Understanding Evaluation,","venue":null,"work_id":"5cf6dce6-c069-421f-8b0e-9590f0bc556e","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.815279Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1de448e60c83cdd9972ef7305d2cad0a913f6aaea86c55f32a850364a2d0f525","observation_id":"b32b011a-7ce8-4541-b6bd-ae0eff8c7655","resolution":{"observed_at":"2026-08-06T13:38:07.663782Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.634024Z","title":"LAraBench: Benchmarking Arabic AI with Large Language Models,","venue":null,"work_id":"860f2470-5e0c-4177-89cc-a7fcf2ae698a","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.258663Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3765495123fec6e38802d7a2eb72a1c9d62e7215a75a2a6047ee675d8d825b66","observation_id":"aa3195d2-a51d-4dc0-b2af-9a6328055803","resolution":{"observed_at":"2026-08-06T13:38:07.638619Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1502.05698","last_updated":"2015-12-31T13:08:14Z","snapshot_observed_at":"2026-07-06T04:09:45.600447Z","submitted_at":"2015-02-19T20:46:10Z","title":"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks","version":10},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1502.05698","snapshot_observed_at":"2026-08-06T13:37:59.448235Z","title":"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.448235Z"},"links":{"cited_paper":"/paper/1502.05698","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:18822729e8fbfa5fb10b6ca1e26f0446ebb48dd5facfb1ffc060b708faf4b22f","observation_id":"d53b659a-8b8c-4b69-b0d3-297b7cee87ca","resolution":{"observed_at":"2026-08-06T13:37:59.448235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.160290Z","title":"ArabicMMLU: Assessing Massive Multitask Language Understanding in Arabic,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.160290Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:afd217814d32d45cafed67bc6c710aab3bb12e0477a928d3fd5b777fc3df7f08","observation_id":"a748ce81-d50b-4170-9c70-a9b4a5157d34","resolution":{"observed_at":"2026-08-06T13:37:59.160290Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2020.findings-emnlp.39","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:02.434165Z","title":"KorNLI and KorSTS: New Benchmark Datasets for Korean Natural Language Understanding,","venue":null,"work_id":"32543dea-2333-4d26-b14d-c05bef0db2f8","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.675564Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:44452064a920874cff07c9f762d3d12961617f91d0de142b1d9270fe76b9d0bb","observation_id":"9877f715-1110-4b79-8c3f-227868f75d03","resolution":{"observed_at":"2026-08-06T13:38:02.509856Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.782498Z","title":"CLUE: A Chinese Language Understanding Evaluation Benchmark,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.782498Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:770a1bdedaf9cf9887fbadfa0732a5ac9ab0284c2fcd16b91a19ebdd94b0e0e2","observation_id":"9a5d9f2e-eccc-4638-a16a-58b6e0d13866","resolution":{"observed_at":"2026-08-06T13:37:59.782498Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.592642Z","title":"JGLUE: Japanese General Language Understanding Evaluation,","venue":null,"work_id":"f8808623-3a4d-4b01-b296-a60290706471","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.854991Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3c915be7b37167f16fd0242bf66e1dee951daafd88618c4a27a0e84ff3adc6dc","observation_id":"3c0eb517-fdff-4864-8015-0a2649c2db02","resolution":{"observed_at":"2026-08-06T13:38:07.597294Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.606742Z","title":"FlauBERT: Unsupervised Language Model Pre-training for French,","venue":null,"work_id":"be269009-c465-4feb-b746-b7abe9e2f282","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.569273Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:23df0c7cf11b7152dc83736e27f99cab8d3d6a5978fab4ebc08c4516e1cc764d","observation_id":"1a814e55-d8bb-433c-aca1-1e236472a633","resolution":{"observed_at":"2026-08-06T13:38:07.611359Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.579262Z","title":"KLUE: Korean Language Understanding Evaluation,","venue":null,"work_id":"2aa97214-e283-4dae-b14f-f028a121e1a7","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.038452Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:534e210464c2026664c0cf6468eff23cdf499631524d64cf34ef46a8aaf192e6","observation_id":"41de1071-0512-47c3-a8fe-401ca88f2706","resolution":{"observed_at":"2026-08-06T13:38:07.583677Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.acl-demo.33","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:02.265909Z","title":"UINAUIL: A Unified Benchmark for Italian Natural Language Understanding,","venue":null,"work_id":"2266952a-ab46-4eb4-b8d8-d200c03736b5","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.166860Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c8735e5402a7b1aa3de6c3e95549dcb386d4b6a3f646c09956d7d94443f21a8e","observation_id":"1f389aa9-9204-45a9-b2f4-5567479ee156","resolution":{"observed_at":"2026-08-06T13:38:02.321727Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:00.244011Z","title":"SuperGLEBer: German Language Understanding Evaluation Benchmark,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.244011Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:73d950b338fd3db04b710622c0939f6df8083509b952bb2df2d599d91d5f5d24","observation_id":"f7a1f880-dcd6-4806-b4b3-97735c014b95","resolution":{"observed_at":"2026-08-06T13:38:00.244011Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.916339Z","title":"IndoNLU: Benchmark and Resources for Evaluating Indonesian Natural Language Understanding,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.916339Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:f72abe209b69ea36d2a1cf5c472493f7e0ef0f8fe08524d809acf8ebd637b9bd","observation_id":"8d1de291-c078-4bfd-a723-c1c8c9e3acb6","resolution":{"observed_at":"2026-08-06T13:37:59.916339Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.565594Z","title":"IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages,","venue":null,"work_id":"4fd9abd1-70cd-4e75-8042-77b5293c3959","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.370586Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7412d423643ba582a9e49b022e76e0d14210bf2e927b87e1fbdab4c182e4e35a","observation_id":"76900af7-102a-4306-8a0b-67ccdec9f9c5","resolution":{"observed_at":"2026-08-06T13:38:07.569610Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.551707Z","title":"Using the Framework,","venue":null,"work_id":"eb2505a6-270d-4917-a1c6-4c9e045ed693","year":1996},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.526646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c8736715c3b0fb4979a032a6693fd691cd4afea2a41c7437706497f62b85e737","observation_id":"1e878b29-8a63-4574-a305-5822fc162d67","resolution":{"observed_at":"2026-08-06T13:38:07.555603Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.537345Z","title":"An extended model of natural logic","venue":null,"work_id":"8d21125b-403a-4c7f-ae67-e16926ea2c78","year":null},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.643216Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:677021dd54a4af6c8d9956c14d10d8cf0b06d60340c56323b4c7e09190e7d3af","observation_id":"ba872268-3e9f-40d9-9560-5bbaf7689c15","resolution":{"observed_at":"2026-08-06T13:38:07.542099Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.findings-naacl.15","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:02.121764Z","title":"VLUE: A New Benchmark and Multi-task Knowledge Transfer Learning for Vietnamese Natural Language Understanding,","venue":null,"work_id":"8e2a3576-4b63-4c3b-a861-582dc9ba6808","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":84,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.314390Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:258d1f1d5f371470af802b756b1896657caa7a083884e317ccb1476b54e4c502","observation_id":"200d0496-5d0a-4469-b32f-05bc9bee392d","resolution":{"observed_at":"2026-08-06T13:38:02.179538Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:05.304873Z","title":"Analysis of identifying linguistic phenomena for recognizing inference in text,","venue":null,"work_id":"d5ca1bb0-861e-4651-a212-c7920c7cf8b8","year":2014},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.819109Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:bacf1624fb11a16ee7be36061d54bf61b2034e843dcac385ebc2a8a65b157fff","observation_id":"b17cb167-b484-401e-a616-35348e3c564f","resolution":{"observed_at":"2026-08-06T13:38:05.500337Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/k19-1033","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:01.878933Z","title":"EQUATE: A Benchmark Evaluation Framework for Quantitative Reasoning in Natural Language Inference,","venue":null,"work_id":"877a590d-b6cd-4a6a-b131-8a77480af1fd","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":86,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.875892Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:00e0da0fb8f2f20883cb236fba1a710b795e4ce11207cb6a4c2959909de7374e","observation_id":"2351178e-b1e2-4ac3-aae2-2b68cae1df7e","resolution":{"observed_at":"2026-08-06T13:38:01.970328Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/p17-1075","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:01.687750Z","title":"Evaluation Metrics for Machine Reading Comprehension: Prerequisite Skills and Readability,","venue":null,"work_id":"5f7219ca-709c-40fb-87ab-760db5df550e","year":2017},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.932270Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:38213f40dd073ff5b53d962f0065c92d5816a125a65b30bfdd06c832ea5ed128","observation_id":"40b6093e-fe1b-4e39-b590-8aa0b747a202","resolution":{"observed_at":"2026-08-06T13:38:01.777765Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:06.975742Z","title":"NaturalLI: Natural Logic Inference for Common Sense Reasoning,","venue":null,"work_id":"983b7ebc-d11b-4ca8-95cb-1e923135ee03","year":2014},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":88,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.991401Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:49c9fe42601cf20d68afaaa005a0ee977187d3a72a6f15eb200a95cb1c3981dd","observation_id":"acffebb4-f9ff-4e31-b663-cfa23187988a","resolution":{"observed_at":"2026-08-06T13:38:07.125392Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.293552Z","title":"Building Textual Entailment Specialized Data Sets: a Methodology for Isolating Linguistic Phenomena Relevant to Inference.,","venue":null,"work_id":"81ba5cd1-7cda-4f15-9d93-deaf8637f64d","year":2010},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":89,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.733334Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:872a862a23e670889aef820145af48ff46c00c82c21a7d1567f36065f28a5084","observation_id":"5dec987b-ab00-4b50-ba09-6af7a885574b","resolution":{"observed_at":"2026-08-06T13:38:07.450314Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.03256","last_updated":"2022-10-13T22:54:22Z","snapshot_observed_at":"2026-08-01T15:58:06.102904Z","submitted_at":"2022-10-06T23:39:01Z","title":"Not another Negation Benchmark: The NaN-NLI Test Suite for Sub-clausal Negation","version":2},"cited_work":{"arxiv_id":"2210.03256","doi":"10.48550/arxiv.2210.03256","metadata_source":"pith","pith_arxiv_id":"2210.03256","snapshot_observed_at":"2026-08-06T18:16:30.548575Z","title":"Not another Negation Benchmark: The NaN-NLI Test Suite for Sub-clausal Negation","venue":"cs.CL","work_id":"c905e041-292b-4e3a-b290-3c0109784fb2","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":90,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.153777Z"},"links":{"cited_paper":"/paper/2210.03256","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:573de48f6514ae7b9fa97f0c0c6fcc53ffec14caedd6e143de68955eacdff317","observation_id":"1106c391-1b24-4171-b2f6-dfb5fc5e9610","resolution":{"observed_at":"2026-08-06T13:38:01.441104Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:06.611565Z","title":"Neural Networks and Textual Inference: How did we get here and where do we go now?,","venue":null,"work_id":"1b5b9a4a-0c00-4d00-846b-99dcccb5dbc0","year":2017},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":91,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.264257Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2f62ea6813c00b9bb8c87c1c25ea76b77580e92e04e3df79f9b41b1955601676","observation_id":"c4943d82-5393-44fc-b3cd-d9f8ea6645d3","resolution":{"observed_at":"2026-08-06T13:38:06.760253Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/n18-2082","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:01.548124Z","title":"On the Evaluation of Semantic Phenomena in Neural Machine Translation Using Natural Language Inference,","venue":null,"work_id":"b9536499-dcd6-4300-a05a-7361790c33d4","year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.049741Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:429fef4a0f62d5d8e3e0bff261a9c8d2e371c04daf17227f03a6deeb17e92272","observation_id":"8fdf345c-55e3-41bd-8aec-bfd2f400b9e4","resolution":{"observed_at":"2026-08-06T13:38:01.605861Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.620554Z","title":"Available: https://aclanthology.org/2024.eacl-long.30/","venue":null,"work_id":"766e2bdf-923c-4aa7-be25-023f5e97628f","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":520,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.366997Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:4f8f69bf87a0a24d2a69ba781313a67d64388b35b3ffbcdff26c7de03cfffe46","observation_id":"40049495-c160-45a7-b31a-d5732925ded0","resolution":{"observed_at":"2026-08-06T13:38:07.625044Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.304442Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":1422,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.304442Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e589ea1512cfda887b5b6ecce762c98b547cb9f683f66e2518835f8451067b48","observation_id":"066998a0-309a-4784-bc5e-efbdddc4b793","resolution":{"observed_at":"2026-08-06T13:37:57.304442Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:00.432949Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":4961,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.432949Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:9acaf4dd556533f83d675bd7687b4790d6ee54a05f08e955dc14085c4f5af2c8","observation_id":"f7cab572-fdec-491a-aa9c-c7017d1b647a","resolution":{"observed_at":"2026-08-06T13:38:00.432949Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.401646Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":6018,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.401646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:956943411da630f655df11ff5038231e7602422f7e4ae95adf113ad60da31bab","observation_id":"b815d477-6ea3-4d30-bf6b-0f182e0c0899","resolution":{"observed_at":"2026-08-06T13:37:58.401646Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-06T13:37:49.830850Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?"},"reference_resolution":{"displayed":94,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":47,"verified_exact":20,"verified_fuzzy":25},"total_outbound_references":94},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 7 August 2026, this Paper Citation Record lists 94 of 94 outbound references and 0 inbound Pith citation observations for arXiv:2507.20419."}