{"as_of":"2026-08-17T10:11:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:8d40689caa9d7c2ac62d75e7e346d622586e87e5682e8725326833e371dedd08","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T12:01:31.461050Z","state":"measured"},{"denominator":28,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":28,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-08T16:21:18.483029Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-11T18:16:11.503799Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"cited_work":{"arxiv_id":"2504.14039","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2504.14039","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"https://arxiv.org/ abs/2504.14039","venue":null,"work_id":"f41fd5af-05c5-476e-a5aa-3198863e2d23","year":2025},"citing_paper":{"arxiv_id":"2605.05424","last_updated":"2026-05-06T20:33:52Z","snapshot_observed_at":"2026-08-11T19:05:25.365570Z","submitted_at":"2026-05-06T20:33:52Z","title":"Evaluating the Reliability of Multiple Large Language Models in Risk Assessment: A CIS Controls Based Approach","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-05-08T16:21:18.483029Z"},"links":{"cited_paper":"/paper/2504.14039","citing_paper":"/paper/2605.05424"},"observation_digest":"sha256:22dc89e38d81111d45edfa84af3b6be03a754e6ae67ed35b9e9b9db3168aba05","observation_id":"1873b43c-28bf-4f49-85f1-1054990fea01","resolution":{"observed_at":"2026-05-11T18:16:11.511985Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2504.14039/citation-record","integrity":"/paper/2504.14039/integrity","json":"/paper/2504.14039/citation-record.json","paper":"/paper/2504.14039"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2202.01279","last_updated":"2022-03-29T16:37:47Z","snapshot_observed_at":"2026-08-16T17:24:01.303787Z","submitted_at":"2022-02-02T20:48:54Z","title":"PromptSource: An Integrated Development Environment and Repository for Natural Language Prompts","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.01279","snapshot_observed_at":"2026-08-16T12:01:31.336619Z","title":"H., Sanh, V ., Yong, Z","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.336619Z"},"links":{"cited_paper":"/paper/2202.01279","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:ab130ac5ee7d9ea22a87b80684e94afc227da0d29f12b433f8775f68167e91b8","observation_id":"243db047-b1bd-4a26-bc73-269e32d69c53","resolution":{"observed_at":"2026-08-16T12:01:31.336619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.04724","last_updated":"2023-12-07T22:07:54Z","snapshot_observed_at":"2026-08-16T14:36:40.398299Z","submitted_at":"2023-12-07T22:07:54Z","title":"Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.04724","snapshot_observed_at":"2026-08-16T12:01:31.343187Z","title":"P., Kapil, D., Kozyrakis, Y ., LeBlanc, D., Milazzo, J., Straumann, A., Synnaeve, G., V on- timitta, V ., Whitman, S., and Saxe, J","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.343187Z"},"links":{"cited_paper":"/paper/2312.04724","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:8b45c707e74295c5b33a311302efa9be2397e7fa0fa9be628213d3d97879c2a0","observation_id":"9c51126b-cc54-49cb-9df2-06b5469e42f7","resolution":{"observed_at":"2026-08-16T12:01:31.343187Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.13161","last_updated":"2024-04-19T20:11:12Z","snapshot_observed_at":"2026-08-16T13:59:21.695627Z","submitted_at":"2024-04-19T20:11:12Z","title":"CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.13161","snapshot_observed_at":"2026-08-16T12:01:31.348307Z","title":"Cyberseceval 2: A wide-ranging cybersecurity evaluation suite for large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.348307Z"},"links":{"cited_paper":"/paper/2404.13161","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:1956b3ae56700e13b754f0d75b0bfb937b20ca12cee82aa8c2dec74be63fd0ed","observation_id":"e2430c4b-1e30-4de9-b5a6-fa571ef65659","resolution":{"observed_at":"2026-08-16T12:01:31.348307Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.20441","last_updated":"2024-10-30T14:29:37Z","snapshot_observed_at":"2026-08-16T13:47:35.143630Z","submitted_at":"2024-05-30T19:35:06Z","title":"SECURE: Benchmarking Large Language Models for Cybersecurity","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.20441","snapshot_observed_at":"2026-08-16T12:01:31.353139Z","title":"T., Nguyen, L., Mahara, A., Lightcap, Z., Frazier, R., Fieblinger, R., Torales, G","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.353139Z"},"links":{"cited_paper":"/paper/2405.20441","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:21d98400361b2bb5af4d8a78ed193024b8a5a324027bec8887e24caa74db9b71","observation_id":"3a87dbfc-f786-4f7c-a054-802783b232cb","resolution":{"observed_at":"2026-08-16T12:01:31.353139Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14782","last_updated":"2026-05-31T00:04:33Z","snapshot_observed_at":"2026-08-16T13:50:06.099874Z","submitted_at":"2024-05-23T16:50:49Z","title":"Lessons from the Trenches on Reproducible Evaluation of Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14782","snapshot_observed_at":"2026-08-16T12:01:31.358076Z","title":"F., Ammanamanchi, P","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.358076Z"},"links":{"cited_paper":"/paper/2405.14782","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:9e3935737bdcde2d510a97f9e1790fc426e2f28c06671dfbc96d3fcdc984f1eb","observation_id":"0c38c05b-8733-47b6-8c6d-18a5a200348c","resolution":{"observed_at":"2026-08-16T12:01:31.358076Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:31.363329Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.363329Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:5c4d03eb4fdbcdf8ccc34bf3e2c220ca38635c8c63fb5d2eaad6da691575892a","observation_id":"313abe81-bff7-4440-8cac-2b226a726fee","resolution":{"observed_at":"2026-08-16T12:01:31.363329Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.09983","last_updated":"2023-10-19T12:41:12Z","snapshot_observed_at":"2026-08-16T15:23:16.542270Z","submitted_at":"2023-06-16T17:26:38Z","title":"Evaluating Superhuman Models with Consistency Checks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.09983","snapshot_observed_at":"2026-08-16T12:01:31.368451Z","title":"Evaluating superhuman models with consistency checks, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.368451Z"},"links":{"cited_paper":"/paper/2306.09983","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:c204fea0d1de87aa7d3f218dce8135b36422931eb652efb66eab089beed916a1","observation_id":"88dcf26a-c69d-4da8-9115-19410e580af6","resolution":{"observed_at":"2026-08-16T12:01:31.368451Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:31.373183Z","title":"A framework for few-shot language model evaluation, 12 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.373183Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:f8a6433746aeb660541f08d94dea6d3caad17c15c78b6a22295ca0df266c598c","observation_id":"acff226d-cf73-4073-8d60-97dd577a17d3","resolution":{"observed_at":"2026-08-16T12:01:31.373183Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1912.05511","last_updated":"2021-03-12T21:26:55Z","snapshot_observed_at":"2026-08-14T12:13:03.207438Z","submitted_at":"2019-12-11T18:21:38Z","title":"Measurement and Fairness","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1912.05511","snapshot_observed_at":"2026-08-16T12:01:31.377637Z","title":null,"venue":null,"work_id":null,"year":1912},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.377637Z"},"links":{"cited_paper":"/paper/1912.05511","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:331209a1ee3330f639111b44eeced30616fe4695d15bd48868fbf02821382e20","observation_id":"09d3bdb1-28b8-4a23-90a8-a4f6c45bb64e","resolution":{"observed_at":"2026-08-16T12:01:31.377637Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.03446","last_updated":"2024-06-03T04:04:52Z","snapshot_observed_at":"2026-08-16T13:55:03.387649Z","submitted_at":"2024-05-06T13:17:43Z","title":"SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.03446","snapshot_observed_at":"2026-08-16T12:01:31.382441Z","title":"Sevenllm: Benchmarking, eliciting, and enhancing abilities of large language models in cyber threat intelligence, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.382441Z"},"links":{"cited_paper":"/paper/2405.03446","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:9b63fda8f28e5762aeb11863bd5d6a73b6f6f5bd4548809ef81e4a9ec5c2735a","observation_id":"3ceda8b9-4bb1-46af-9560-e7485d4d4f86","resolution":{"observed_at":"2026-08-16T12:01:31.382441Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:32.001899Z","title":"Seceval: A comprehensive benchmark for eval- uating cybersecurity knowledge of foundation models","venue":null,"work_id":"86337a01-8976-474f-8dce-39197afa0a38","year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.387051Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:3c27925fc768f4287093b4dfa79781f054e2b0182ad0c92d9ea79e89792ba431","observation_id":"d05a05ba-6c23-4f4f-9569-6de44eb14441","resolution":{"observed_at":"2026-08-16T12:01:32.006642Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03218","last_updated":"2024-05-15T19:16:09Z","snapshot_observed_at":"2026-08-15T06:18:34.739211Z","submitted_at":"2024-03-05T18:59:35Z","title":"The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.03218","snapshot_observed_at":"2026-08-16T12:01:31.391346Z","title":"D., Dombrowski, A.-K., Goel, S., Phan, L., Mukobi, G., Helm-Burger, N., Lababidi, R., Justen, L., Liu, A","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.391346Z"},"links":{"cited_paper":"/paper/2403.03218","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:0cd5677175c2f54895897faabd4ecdc80eb0ac1936ce4385b45cf25bca4373a0","observation_id":"cf77ecd3-503e-4bc6-aa72-f110d08e7625","resolution":{"observed_at":"2026-08-16T12:01:31.391346Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:31.985585Z","title":"ROUGE: A package for automatic evaluation of summaries","venue":null,"work_id":"5fa88f0c-9a82-40fb-83b8-7a0d7d2e9401","year":2004},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.396250Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:c0021d5d43f31af45e8b46dbdbe032ed35fff6eb572268ba4aad180fb28568b6","observation_id":"40a79429-96d1-48da-961d-a9b8afb933b7","resolution":{"observed_at":"2026-08-16T12:01:31.990460Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.15838","last_updated":"2023-12-26T00:59:30Z","snapshot_observed_at":"2026-08-16T14:31:41.602497Z","submitted_at":"2023-12-26T00:59:30Z","title":"SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.15838","snapshot_observed_at":"2026-08-16T12:01:31.400840Z","title":"Secqa: A concise question-answering dataset for evaluating large language models in computer security, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.400840Z"},"links":{"cited_paper":"/paper/2312.15838","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:8224f55b3dc41151f764b7793a46e88a38f4597c98330d0e0c696a8f60380de4","observation_id":"89da18fe-18d3-4128-ab03-f233e45f5cc0","resolution":{"observed_at":"2026-08-16T12:01:31.400840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.04249","last_updated":"2024-02-27T04:43:08Z","snapshot_observed_at":"2026-08-16T09:07:20.265665Z","submitted_at":"2024-02-06T18:59:08Z","title":"HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.04249","snapshot_observed_at":"2026-08-16T12:01:31.405565Z","title":"Harmbench: A standardized evaluation frame- work for automated red teaming and robust refusal, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.405565Z"},"links":{"cited_paper":"/paper/2402.04249","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:1443b39be193b7cf93f03c5975ebc8ac62b11f504eabfd7fa491c3967074d221","observation_id":"692c1e35-aa40-4370-8630-f4ad00f0e8bf","resolution":{"observed_at":"2026-08-16T12:01:31.405565Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.00595","last_updated":"2024-05-06T10:20:26Z","snapshot_observed_at":"2026-08-16T14:30:34.654390Z","submitted_at":"2023-12-31T22:21:36Z","title":"State of What Art? A Call for Multi-Prompt LLM Evaluation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.00595","snapshot_observed_at":"2026-08-16T12:01:31.410067Z","title":"State of what art? a call for multi-prompt llm evaluation, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.410067Z"},"links":{"cited_paper":"/paper/2401.00595","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:940c2fdba80080cde330b823d216a1a78efb718416d57e4c44486ed87bf11be1","observation_id":"13e0382f-c17c-4226-8a95-c333b5fe7729","resolution":{"observed_at":"2026-08-16T12:01:31.410067Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21792","last_updated":"2024-12-27T17:36:21Z","snapshot_observed_at":"2026-08-16T13:29:26.927805Z","submitted_at":"2024-07-31T17:59:24Z","title":"Safetywashing: Do AI Safety Benchmarks Actually Measure Safety Progress?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21792","snapshot_observed_at":"2026-08-16T12:01:31.414686Z","title":"H., Fitz, S., and Hendrycks, D","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.414686Z"},"links":{"cited_paper":"/paper/2407.21792","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:9e7a783ef6c7be1b8628134218d9d2316ec623a98abe92f815b722eeed9dc342","observation_id":"807142c3-38b0-45f1-b62e-61073d9f8e5e","resolution":{"observed_at":"2026-08-16T12:01:31.414686Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:31.970202Z","title":"Jinja2 Documentation, 2024","venue":null,"work_id":"c627984f-aaaf-45d5-9915-52f3463f72ad","year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.419032Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:0a8220f98974d15fcfe38e3f73532327a9db49fa9330d91857c86f36c23f3901","observation_id":"b1e30fb3-fd28-4cf1-bb9d-2484d3b7cd8f","resolution":{"observed_at":"2026-08-16T12:01:31.974843Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.05020","last_updated":"2024-03-26T02:23:27Z","snapshot_observed_at":"2026-08-16T14:44:49.437725Z","submitted_at":"2023-11-08T21:13:38Z","title":"First Tragedy, then Parse: History Repeats Itself in the New Era of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.05020","snapshot_observed_at":"2026-08-16T12:01:31.423531Z","title":"First tragedy, then parse: History repeats itself in the new era of large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.423531Z"},"links":{"cited_paper":"/paper/2311.05020","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:ced851f588731cafe62048ca72d81d8534553014c3905ddf5e57143e1c215eb9","observation_id":"4bc161a4-aab5-429b-b3e9-b68d7cf27dfe","resolution":{"observed_at":"2026-08-16T12:01:31.423531Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T12:01:31.952985Z","title":"Quantifying language models’ sensitivity to spurious features in prompt design or: How i learned to start worrying about prompt for- matting, 2024","venue":null,"work_id":"2cdbf8b1-48bb-4bd0-b470-67734414dff6","year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.428320Z"},"links":{"citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:ca256531228c398028557a0a64036db11e3f3cc4688a0a8fea1b73804c174116","observation_id":"e52be142-08b5-4129-a223-2cbdae6225f2","resolution":{"observed_at":"2026-08-16T12:01:31.959706Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.01724","last_updated":"2024-05-02T20:42:28Z","snapshot_observed_at":"2026-08-16T13:55:48.904628Z","submitted_at":"2024-05-02T20:42:28Z","title":"Large Language Models are Inconsistent and Biased Evaluators","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.01724","snapshot_observed_at":"2026-08-16T12:01:31.433321Z","title":"Large language models are inconsistent and biased evaluators, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.433321Z"},"links":{"cited_paper":"/paper/2405.01724","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:07baf07fc6503754aa89250c7b2ddfa8ecefa6fc5641522e58fe62652964ad53","observation_id":"c826add8-ad35-48ae-a152-2a3d399737ab","resolution":{"observed_at":"2026-08-16T12:01:31.433321Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.09022","last_updated":"2023-05-15T21:12:07Z","snapshot_observed_at":"2026-08-16T15:32:52.449485Z","submitted_at":"2023-05-15T21:12:07Z","title":"It Takes Two to Tango: Navigating Conceptualizations of NLP Tasks and Measurements of Performance","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.09022","snapshot_observed_at":"2026-08-16T12:01:31.438248Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.438248Z"},"links":{"cited_paper":"/paper/2305.09022","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:d2f62f81b129a408b888b29914d5c00a8817878de0d0da6d8ee6ef7b3ee7ecbe","observation_id":"3c2b4058-a6ac-46fe-b104-ad97b75c80a2","resolution":{"observed_at":"2026-08-16T12:01:31.438248Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07688","last_updated":"2024-06-03T08:14:45Z","snapshot_observed_at":"2026-08-16T14:19:19.666029Z","submitted_at":"2024-02-12T14:53:28Z","title":"CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07688","snapshot_observed_at":"2026-08-16T12:01:31.442971Z","title":"A., Jain, R., Bisztray, T., and Debbah, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.442971Z"},"links":{"cited_paper":"/paper/2402.07688","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:9a27586c7245b74cedfef8e674f853acd2ec98b016a1c1bff15d61b352fe1ed8","observation_id":"0d61b9a3-360a-440f-8743-57679b614f7f","resolution":{"observed_at":"2026-08-16T12:01:31.442971Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14889","last_updated":"2023-10-23T01:02:48Z","snapshot_observed_at":"2026-08-16T15:30:17.203213Z","submitted_at":"2023-05-24T08:38:23Z","title":"Evaluating Evaluation Metrics: A Framework for Analyzing NLG Evaluation Metrics using Measurement Theory","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14889","snapshot_observed_at":"2026-08-16T12:01:31.447787Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.447787Z"},"links":{"cited_paper":"/paper/2305.14889","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:22d7498d8b0ecd3898d448afec90171b0329ad5afc3a9456055f8d2c72fbbd02","observation_id":"a7f2cb32-15ef-4bcf-818e-89b9aaee5a60","resolution":{"observed_at":"2026-08-16T12:01:31.447787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17567","last_updated":"2023-10-26T16:55:05Z","snapshot_observed_at":"2026-08-16T14:48:32.679869Z","submitted_at":"2023-10-26T16:55:05Z","title":"Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17567","snapshot_observed_at":"2026-08-16T12:01:31.452260Z","title":"Skill-mix: a flexible and expandable family of evaluations for ai models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.452260Z"},"links":{"cited_paper":"/paper/2310.17567","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:1b0417303774e1e86440b8580d2e42cf18b189ec34806ee1538f8bcbf77f979c","observation_id":"5b3f79fc-2b08-4467-bf0a-ad11d353c58a","resolution":{"observed_at":"2026-08-16T12:01:31.452260Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-16T12:01:31.456864Z","title":"P., Zhang, H., Gonzalez, J","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.456864Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:987e763d3ff381081ababec4f709e65e54c8c3c3164bb0a6c4ca912fbb1f3d0b","observation_id":"426e0c94-8ccc-4abf-a0c1-bcdde1e06c57","resolution":{"observed_at":"2026-08-16T12:01:31.456864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.17167","last_updated":"2024-03-14T09:52:16Z","snapshot_observed_at":"2026-08-16T14:56:32.958443Z","submitted_at":"2023-09-29T12:04:14Z","title":"DyVal: Dynamic Evaluation of Large Language Models for Reasoning Tasks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.17167","snapshot_observed_at":"2026-08-16T12:01:31.461050Z","title":"canary strings","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-16T12:01:31.461050Z"},"links":{"cited_paper":"/paper/2309.17167","citing_paper":"/paper/2504.14039"},"observation_digest":"sha256:4b1b2e2891dbd785a9ded09617d61d7754b671965abd3885fb8d7d5e2124ee8c","observation_id":"bc37ec53-3f00-410b-9698-170f711e3da8","resolution":{"observed_at":"2026-08-16T12:01:31.461050Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2504.14039","last_updated":"2025-04-18T19:01:53Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-16T11:55:47.679654Z","submitted_at":"2025-04-18T19:01:53Z","title":"MEQA: A Meta-Evaluation Framework for Question & Answer LLM Benchmarks"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":23,"verified_exact":0,"verified_fuzzy":4},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 1 inbound Pith citation observation for arXiv:2504.14039."}