{"as_of":"2026-08-23T15:04:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:a38cf9e9aff62c04394a17ae210a733f6c33062492e61a13094956143fecac16","coverage":[{"denominator":21,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":21,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-11T11:50:26.030339Z","state":"measured"},{"denominator":121,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":121,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-23T06:30:58.430688+00:00","state":"measured"},{"denominator":389,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T12:08:24.813142Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"pith","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":16,"observed_at":"2026-08-05T02:28:24.338817Z","source":"pith"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2211.05100","last_updated":"2023-06-27T09:57:58Z","snapshot_observed_at":"2026-08-04T18:56:03.233715Z","submitted_at":"2022-11-09T18:48:09Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","version":4},"reference_index":187,"source":"arxiv_source","source_observed_at":"2026-05-12T00:51:10.919818Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2211.05100"},"observation_digest":"sha256:011857f799cb14bf0bc72c45a7ed1b990768bc5409322148b1ee2932eeba6f46","observation_id":"2a2e31f5-aeec-4826-b78d-e5c493f93d40","resolution":{"observed_at":"2026-05-12T00:51:11.310415Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2211.05100","last_updated":"2023-06-27T09:57:58Z","snapshot_observed_at":"2026-08-04T18:56:03.233715Z","submitted_at":"2022-11-09T18:48:09Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","version":4},"reference_index":266,"source":"arxiv_source","source_observed_at":"2026-05-12T00:51:10.919818Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2211.05100"},"observation_digest":"sha256:53aafde083c25e1d3e2fdd084eed452fbbd141205541f53024e497adeff185dd","observation_id":"21df8cda-88d8-46fd-8b46-d991c83a6b65","resolution":{"observed_at":"2026-05-12T00:51:11.617142Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2303.17564","last_updated":"2023-12-21T06:21:11Z","snapshot_observed_at":"2026-08-14T12:54:48.492396Z","submitted_at":"2023-03-30T17:30:36Z","title":"BloombergGPT: A Large Language Model for Finance","version":3},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-05-13T23:19:46.231145Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2303.17564"},"observation_digest":"sha256:7d1b11b73ec1077c4106ea9edda59bd0bf8108c347e6cdc553d3878525a56732","observation_id":"b857d7ae-e7a1-4bad-b72b-3dc02cf82a60","resolution":{"observed_at":"2026-05-13T23:19:46.406429Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2304.06364","last_updated":"2023-09-18T14:23:02Z","snapshot_observed_at":"2026-08-07T12:51:50.861720Z","submitted_at":"2023-04-13T09:39:30Z","title":"AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models","version":2},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-05-16T10:03:58.971585Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2304.06364"},"observation_digest":"sha256:96d4fb3037023f9a2e0db4fac907bcc11f97c26d4d062f552482563691c7a260","observation_id":"71fd7756-0d77-4e79-98c8-079e786055e0","resolution":{"observed_at":"2026-05-16T10:03:59.393937Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.04388","last_updated":"2023-12-09T21:25:02Z","snapshot_observed_at":"2026-08-12T21:03:44.597326Z","submitted_at":"2023-05-07T22:44:25Z","title":"Language Models Don't Always Say What They Think: Unfaithful Explanations in Chain-of-Thought Prompting","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.04388"},"observation_digest":"sha256:580bf4fdb0f85b02c4f2fa5ecefed3d70b800c263e03e0fedab6c9ca978805f3","observation_id":"109acafb-4654-4b97-ade5-b7d1d0f50f30","resolution":{"observed_at":"2026-05-15T12:01:19.833619Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.06161","last_updated":"2023-12-13T14:44:10Z","snapshot_observed_at":"2026-07-06T15:25:35.930688Z","submitted_at":"2023-05-09T08:16:42Z","title":"StarCoder: may the source be with you!","version":2},"reference_index":278,"source":"arxiv_source","source_observed_at":"2026-05-10T23:32:59.517389Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.06161"},"observation_digest":"sha256:a96270d8b2fb2b89049d93a37dabc04d735d48177a43a89bc7094cdd1266a033","observation_id":"284311c4-f8f7-4c9e-9863-d59a231203bf","resolution":{"observed_at":"2026-05-10T23:33:01.097348Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.09617","last_updated":"2023-05-16T17:11:29Z","snapshot_observed_at":"2026-08-20T13:33:56.038060Z","submitted_at":"2023-05-16T17:11:29Z","title":"Towards Expert-Level Medical Question Answering with Large Language Models","version":1},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-05-24T04:32:33.271634Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.09617"},"observation_digest":"sha256:d02ebf1de41e422adc98b34eb8d544f5019b3163f5dcaa693ef09db974845e8a","observation_id":"2fac3c34-f5b5-44a0-95d9-1cfb280ef061","resolution":{"observed_at":"2026-05-24T04:32:33.596042Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.10403","last_updated":"2023-09-13T20:35:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-05-17T17:46:53Z","title":"PaLM 2 Technical Report","version":3},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-05-12T11:59:25.813128Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.10403"},"observation_digest":"sha256:e489b14610535a8a5790caef12aed4a9708ecc6c6bd9720697b4ae571a851c99","observation_id":"ccaf4683-caab-4354-85e0-1b4d6528f6c1","resolution":{"observed_at":"2026-05-12T11:59:27.187433Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.14314","last_updated":"2023-05-23T17:50:33Z","snapshot_observed_at":"2026-08-20T16:10:20.713555Z","submitted_at":"2023-05-23T17:50:33Z","title":"QLoRA: Efficient Finetuning of Quantized LLMs","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-05-11T13:29:53.345251Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.14314"},"observation_digest":"sha256:c459739066f580b88324c7b3d4acaaa9bdc78e73f547dd60a0f4970e960cb116","observation_id":"3114b01c-f1c7-4dac-b5ad-6c9c791af106","resolution":{"observed_at":"2026-05-11T13:29:53.548060Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2305.16264","last_updated":"2025-06-28T00:00:06Z","snapshot_observed_at":"2026-08-21T22:49:19.447804Z","submitted_at":"2023-05-25T17:18:55Z","title":"Scaling Data-Constrained Language Models","version":5},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-05-18T01:35:21.150772Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2305.16264"},"observation_digest":"sha256:ac93157b036ccce941eac8f5c2b5ee34e90215577e6e7e22fc8ffd8132f4bfa4","observation_id":"c50b9ac0-713b-4088-a30b-8a22fa424d79","resolution":{"observed_at":"2026-05-18T01:35:21.244425Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-05-10T18:52:59.033645Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2306.05685"},"observation_digest":"sha256:b789ad931a65b3d3cf98274d1e09b1cc7c1b4977180a4bc24df5ae25d110cf9e","observation_id":"4c4b2457-6247-403d-a0b9-40662455db54","resolution":{"observed_at":"2026-05-10T18:52:59.095035Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2306.14048","last_updated":"2023-12-18T19:10:00Z","snapshot_observed_at":"2026-08-20T21:36:51.212852Z","submitted_at":"2023-06-24T20:11:14Z","title":"H$_2$O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-05-17T18:00:50.053377Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2306.14048"},"observation_digest":"sha256:8a1bc8a22757fd8b9d24ee3570d039557c1089f29e61ceee3cc5b733d83ef43f","observation_id":"e7f06c6e-425d-4089-b4da-5e359e40c414","resolution":{"observed_at":"2026-05-17T18:00:50.427058Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2308.05374","last_updated":"2024-03-21T00:21:14Z","snapshot_observed_at":"2026-08-02T17:09:20.540788Z","submitted_at":"2023-08-10T06:43:44Z","title":"Trustworthy LLMs: a Survey and Guideline for Evaluating Large Language Models' Alignment","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-05-17T22:30:44.520703Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2308.05374"},"observation_digest":"sha256:fec9d5b412617dfa73e882962a70b9ee491783f76e337bc00790c5bcd2fe1c5e","observation_id":"4db573e6-6781-430a-b62c-0d42808378ed","resolution":{"observed_at":"2026-05-17T22:30:44.671076Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2311.12983","last_updated":"2023-11-21T20:34:47Z","snapshot_observed_at":"2026-08-13T10:06:26.439949Z","submitted_at":"2023-11-21T20:34:47Z","title":"GAIA: a benchmark for General AI Assistants","version":1},"reference_index":115,"source":"arxiv_source","source_observed_at":"2026-05-12T15:46:03.247029Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2311.12983"},"observation_digest":"sha256:a70ef3591bd51589e172953a69471f7798de80ddaf4f8a7adef6319118251db5","observation_id":"2276c250-7782-4a1b-9da1-8fc07679cd7c","resolution":{"observed_at":"2026-05-12T15:46:03.400665Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2311.16867","last_updated":"2023-11-29T19:45:10Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-11-28T15:12:47Z","title":"The Falcon Series of Open Language Models","version":2},"reference_index":212,"source":"arxiv_source","source_observed_at":"2026-05-16T09:46:09.701440Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2311.16867"},"observation_digest":"sha256:4f13d057fd99dde3cbce81c26e9a08b93a888241e86472c812ea36deb254cbe2","observation_id":"f3811a93-4e13-4ad1-945e-4db5cd664d97","resolution":{"observed_at":"2026-05-16T09:46:09.906965Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2401.05561","last_updated":"2024-09-30T10:17:12Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-01-10T22:07:21Z","title":"TrustLLM: Trustworthiness in Large Language Models","version":6},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-05-18T11:17:08.108565Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2401.05561"},"observation_digest":"sha256:1356f34c1a385d1d0716abc0a947cd96a24835ede22d96b5360852b5959e7c11","observation_id":"dacd4082-fada-42c9-bcd6-eccb45ff3db4","resolution":{"observed_at":"2026-05-18T11:17:08.428332Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2402.13228","last_updated":"2024-07-03T13:46:33Z","snapshot_observed_at":"2026-08-14T02:18:02.339361Z","submitted_at":"2024-02-20T18:42:34Z","title":"Smaug: Fixing Failure Modes of Preference Optimisation with DPO-Positive","version":2},"reference_index":121,"source":"arxiv_source","source_observed_at":"2026-05-17T23:04:44.287660Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2402.13228"},"observation_digest":"sha256:13e3a9ec9fce3b2d15b6ea5a6903f0517f920cde86c199b4e53e8e6e1b53c9cd","observation_id":"24054ef2-9599-4b38-be4d-b42b0b32f8c9","resolution":{"observed_at":"2026-05-17T23:04:44.477071Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2406.01574","last_updated":"2024-11-06T02:54:00Z","snapshot_observed_at":"2026-08-20T16:26:42.002863Z","submitted_at":"2024-06-03T17:53:00Z","title":"MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark","version":6},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-05-11T15:51:04.674346Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2406.01574"},"observation_digest":"sha256:dc7c538e7ea9cba48146b259415a334f3b64a9a06d477f8c806eb9ef67f0a923","observation_id":"8a3635d8-5b20-4c17-9a12-330413744d0b","resolution":{"observed_at":"2026-05-11T15:51:08.467354Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2406.11939","last_updated":"2024-10-14T18:11:58Z","snapshot_observed_at":"2026-08-21T06:07:47.921030Z","submitted_at":"2024-06-17T17:26:10Z","title":"From Crowdsourced Data to High-Quality Benchmarks: Arena-Hard and BenchBuilder Pipeline","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2406.11939"},"observation_digest":"sha256:90f84506ca4948dd2eca5acb9af8a72680169bc3b0fc4b30b38a96ac284a12fe","observation_id":"ca8c94d0-bb92-4d0e-ba17-fd0a9cad37e2","resolution":{"observed_at":"2026-05-24T00:03:38.982197Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2409.00084","last_updated":"2024-09-04T08:22:28Z","snapshot_observed_at":"2026-08-13T14:19:09.832396Z","submitted_at":"2024-08-25T14:50:47Z","title":"Vision-Language and Large Language Model Performance in Gastroenterology: GPT, Claude, Llama, Phi, Mistral, Gemma, and Quantized Models","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-05-23T22:15:52.638622Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2409.00084"},"observation_digest":"sha256:54322a697a702ad83c25b5477219c2c46b047982d77cca10e72d1cad24cdbaf4","observation_id":"d4b8d1bf-352d-402b-b588-85accc1e953a","resolution":{"observed_at":"2026-05-23T22:18:31.890032Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T21:16:35.248965Z","title":"LLaVA-Med: Training a Large Language- and-Vision Assistant for Biomedicine in One Day","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.08870","last_updated":"2025-06-28T21:17:10Z","snapshot_observed_at":"2026-08-15T05:40:09.612647Z","submitted_at":"2024-11-13T18:50:13Z","title":"The Limited Impact of Medical Adaptation of Large Language and Vision-Language Models","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T21:16:35.248965Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.08870"},"observation_digest":"sha256:a3599259026b23b897bb013c5c10c8511b0de603f3515603928ddb40677232e8","observation_id":"4ceacae8-a589-4f00-a092-b3fe1463346b","resolution":{"observed_at":"2026-08-12T21:16:35.248965Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T20:36:02.007205Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.09523","last_updated":"2024-11-14T15:40:04Z","snapshot_observed_at":"2026-08-21T01:11:25.975486Z","submitted_at":"2024-11-14T15:40:04Z","title":"Navigating the Risks: A Survey of Security, Privacy, and Ethics Threats in LLM-Based Agents","version":1},"reference_index":149,"source":"pdf_text","source_observed_at":"2026-08-12T20:36:02.007205Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.09523"},"observation_digest":"sha256:7c499aa8e3d1d18bf40c79b1bf1b051c6f65771f74610aab1bb875b17866cd6f","observation_id":"f666593b-a629-4ee6-9b84-18f6c44e7f36","resolution":{"observed_at":"2026-08-12T20:36:02.007205Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T20:10:31.073347Z","title":"Holis- tic evaluation of language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.09972","last_updated":"2024-11-15T06:05:45Z","snapshot_observed_at":"2026-08-20T01:53:16.352332Z","submitted_at":"2024-11-15T06:05:45Z","title":"Large Language Models as User-Agents for Evaluating Task-Oriented-Dialogue Systems","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T20:10:31.073347Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.09972"},"observation_digest":"sha256:4f3caf3fcc7c3a1621c563a2d0de145c99c53e2c7bbe55e6c6aeb97b9c31c140","observation_id":"1d530560-b6e5-4118-b30a-9f1c0fc96e01","resolution":{"observed_at":"2026-08-12T20:10:31.073347Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T19:31:34.368539Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.10670","last_updated":"2024-11-16T02:16:59Z","snapshot_observed_at":"2026-08-17T14:18:18.096968Z","submitted_at":"2024-11-16T02:16:59Z","title":"IntentGPT: Few-shot Intent Discovery with Large Language Models","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-12T19:31:34.368539Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.10670"},"observation_digest":"sha256:570219ab500996050b2e9ac3901c70bd3b76977c9d98ab65ec3dac4dcf6e5a1b","observation_id":"a8a115a4-1bc7-414d-8a1c-72726c832982","resolution":{"observed_at":"2026-08-12T19:31:34.368539Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T18:53:58.655265Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.11217","last_updated":"2024-11-18T01:06:12Z","snapshot_observed_at":"2026-08-18T20:24:28.154075Z","submitted_at":"2024-11-18T01:06:12Z","title":"MoE-Lightning: High-Throughput MoE Inference on Memory-constrained GPUs","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T18:53:58.655265Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.11217"},"observation_digest":"sha256:02104baa96d8c690777089b3d234737b0e3352bf203f81f3854ceb9e8db5dae4","observation_id":"854a1e9f-aa4b-4011-9d9e-4a05206611dc","resolution":{"observed_at":"2026-08-12T18:53:58.655265Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T17:15:21.919619Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.12709","last_updated":"2024-11-19T18:25:30Z","snapshot_observed_at":"2026-08-16T03:04:10.336968Z","submitted_at":"2024-11-19T18:25:30Z","title":"Dimensions of Generative AI Evaluation Design","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T17:15:21.919619Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.12709"},"observation_digest":"sha256:a1a481c59acaac924128860aefa528dbbfc09fe724981bac2c99eb87b0a17b43","observation_id":"6effd4b1-d362-43ff-9d01-695d8a0137ea","resolution":{"observed_at":"2026-08-12T17:15:21.919619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T20:13:57.647383Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13577","last_updated":"2024-11-26T09:20:48Z","snapshot_observed_at":"2026-08-20T13:09:54.981053Z","submitted_at":"2024-11-15T04:16:45Z","title":"WavChat: A Survey of Spoken Dialogue Models","version":2},"reference_index":126,"source":"pdf_text","source_observed_at":"2026-08-12T20:13:57.647383Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.13577"},"observation_digest":"sha256:f4ba82b7575c7d143ccec6e4d37481e63bc96e5e4a77d9001508191b2b5bcd49","observation_id":"8b535e1e-46e6-4529-b7dc-dfb5862ecb3a","resolution":{"observed_at":"2026-08-12T20:13:57.647383Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T15:54:33.121732Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13808","last_updated":"2024-11-21T03:14:31Z","snapshot_observed_at":"2026-08-20T07:37:12.375785Z","submitted_at":"2024-11-21T03:14:31Z","title":"GPAI Evaluations Standards Taskforce: Towards Effective AI Governance","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T15:54:33.121732Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.13808"},"observation_digest":"sha256:eb10f1e039996bf3566413c9e78d56a191a8b82c87cff81e12a70ce009c3229b","observation_id":"c4e1c07a-6472-4e33-8bce-b595859eb664","resolution":{"observed_at":"2026-08-12T15:54:33.121732Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T16:56:44.738374Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.14487","last_updated":"2024-11-20T06:34:32Z","snapshot_observed_at":"2026-08-14T06:48:26.798254Z","submitted_at":"2024-11-20T06:34:32Z","title":"Ensuring Safety and Trust: Analyzing the Risks of Large Language Models in Medicine","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T16:56:44.738374Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.14487"},"observation_digest":"sha256:267777dabfbf51ff0e61f2d70c7b0f5928d5dcabe72161bd0cc362ee7a0c2440","observation_id":"27f63275-d104-41ff-9613-56c2c419aae4","resolution":{"observed_at":"2026-08-12T16:56:44.738374Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T14:29:54.226325Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-17T20:54:25.584082Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.226325Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:2e557073a68a48510aa74c10ed8871318cea93fa1d5697f0584a793fc7b08eb6","observation_id":"45e17965-2ffe-431c-bd3b-2ba97c179f1d","resolution":{"observed_at":"2026-08-12T14:29:54.226325Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T14:27:57.413062Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-20T16:59:05.405072Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.413062Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:75de5f6fd00cbbd6785451a67ab316f91c714b5021893084f68aff0be081e3fc","observation_id":"dbf6de75-1d22-4c99-acec-3c2b57599080","resolution":{"observed_at":"2026-08-12T14:27:57.413062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T13:27:06.690101Z","title":"https://github.com/XuanwuAI/SecEval","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.16239","last_updated":"2025-01-17T04:19:43Z","snapshot_observed_at":"2026-08-13T11:46:24.476166Z","submitted_at":"2024-11-25T09:54:42Z","title":"CS-Eval: A Comprehensive Large Language Model Benchmark for CyberSecurity","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T13:27:06.690101Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.16239"},"observation_digest":"sha256:08d2f740e7253a6feb1f55a0c566b3c9bc624464ef0255b053d32f1ff6084e31","observation_id":"38462fc9-d789-48ce-8a39-a724681605a7","resolution":{"observed_at":"2026-08-12T13:27:06.690101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T12:28:14.337637Z","title":", author Bommasani, R","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17204","last_updated":"2024-11-28T01:04:40Z","snapshot_observed_at":"2026-08-21T05:23:18.008009Z","submitted_at":"2024-11-26T08:21:24Z","title":"Strategic Prompting for Conversational Tasks: A Comparative Analysis of Large Language Models Across Diverse Conversational Tasks","version":2},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-12T12:28:14.337637Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.17204"},"observation_digest":"sha256:b1da7a651591f70cff1dd2d1e470fc0e9d3ea2165dd95cdeeb06e6142b66a38f","observation_id":"e4b60683-06f7-44b2-9629-5734beb4a3c3","resolution":{"observed_at":"2026-08-12T12:28:14.337637Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T12:21:33.895762Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17338","last_updated":"2024-11-26T11:32:43Z","snapshot_observed_at":"2026-08-19T01:33:08.375321Z","submitted_at":"2024-11-26T11:32:43Z","title":"Different Bias Under Different Criteria: Assessing Bias in LLMs with a Fact-Based Approach","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T12:21:33.895762Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.17338"},"observation_digest":"sha256:022cad80ab6f6cb4fc2d09bb56eb5d6ebf2eb949ed702bcc2993886efc0ee238","observation_id":"7ffce77d-4554-48c2-b7a0-00122bd02305","resolution":{"observed_at":"2026-08-12T12:21:33.895762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T05:33:56.136541Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.00353","last_updated":"2024-12-06T10:24:47Z","snapshot_observed_at":"2026-08-19T02:28:30.681637Z","submitted_at":"2024-11-30T04:22:00Z","title":"Enhancing Zero-shot Chain of Thought Prompting via Uncertainty-Guided Strategy Selection","version":2},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-12T05:33:56.136541Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.00353"},"observation_digest":"sha256:c66f8394c80955d535f17b2e233cf4c690468dac78d078f261e9050765a462d0","observation_id":"4e7290a1-7f3f-4dad-81a5-8aaeb574c22d","resolution":{"observed_at":"2026-08-12T05:33:56.136541Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T05:21:55.117620Z","title":"arXiv preprint arXiv:2211.09110 (2022)","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.00546","last_updated":"2024-11-30T17:39:59Z","snapshot_observed_at":"2026-08-19T12:38:36.996989Z","submitted_at":"2024-11-30T17:39:59Z","title":"Rank It, Then Ask It: Input Reranking for Maximizing the Performance of LLMs on Symmetric Tasks","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-12T05:21:55.117620Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.00546"},"observation_digest":"sha256:37f25d1c9e42b4d80f3fe4ebc909d51e2ef16c913c913fde3cb67478f0c6fb87","observation_id":"63c82884-8e57-4f5a-8d98-b09919060252","resolution":{"observed_at":"2026-08-12T05:21:55.117620Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T05:18:55.812296Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.00554","last_updated":"2024-12-07T19:00:45Z","snapshot_observed_at":"2026-08-15T10:39:56.350007Z","submitted_at":"2024-11-30T18:52:30Z","title":"Unveiling Performance Challenges of Large Language Models in Low-Resource Healthcare: A Demographic Fairness Perspective","version":2},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-12T05:18:55.812296Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.00554"},"observation_digest":"sha256:33e181913a0eaf55ea8ba8417b1796db03e15188cf6c19adf782f05b0816e7e5","observation_id":"33d99f40-47cc-4c61-aea0-6812515a4d73","resolution":{"observed_at":"2026-08-12T05:18:55.812296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T04:35:59.502032Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.01233","last_updated":"2024-12-02T07:54:55Z","snapshot_observed_at":"2026-08-15T22:41:09.800294Z","submitted_at":"2024-12-02T07:54:55Z","title":"Best Practices for Large Language Models in Radiology","version":1},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-12T04:35:59.502032Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.01233"},"observation_digest":"sha256:f895912dd7bff9136d7b0884c2ec27bfa34f38fe446620d82cbf4124b59fd7c8","observation_id":"627e066d-cf78-46da-beef-dbc4afd38e81","resolution":{"observed_at":"2026-08-12T04:35:59.502032Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T04:20:21.483043Z","title":"Holistic Evaluation of Language Models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-14T23:47:43.827436Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.483043Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:9fc427d560ce4e029d5aa167dacf5ae32b0893d189ebee797dc3a8678db7f7c6","observation_id":"ffd2df8d-b59f-4695-9cc1-eb5b565deece","resolution":{"observed_at":"2026-08-12T04:20:21.483043Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T04:22:02.813497Z","title":"Holistic evalu- ation of language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.01528","last_updated":"2025-08-21T05:56:47Z","snapshot_observed_at":"2026-08-17T23:56:16.045921Z","submitted_at":"2024-12-02T14:19:44Z","title":"CopyrightShield: Enhancing Diffusion Model Security against Copyright Infringement Attacks","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T04:22:02.813497Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.01528"},"observation_digest":"sha256:f8276ad5ce815a7005283c1da817a5b417076ceeb31b46c253a6b84bfadbd736","observation_id":"dc408e55-2172-4b16-ae57-05e086bfd276","resolution":{"observed_at":"2026-08-12T04:22:02.813497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T22:06:51.303089Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.03822","last_updated":"2024-12-05T02:35:46Z","snapshot_observed_at":"2026-08-18T04:19:06.016742Z","submitted_at":"2024-12-05T02:35:46Z","title":"Beyond the Binary: Capturing Diverse Preferences With Reward Regularization","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-11T22:06:51.303089Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.03822"},"observation_digest":"sha256:8935fdba3b97ec6393449486fefb7b4d49de6a7d643ac70009ca53778eab2700","observation_id":"396228b2-8607-415e-a327-5bec2dd767af","resolution":{"observed_at":"2026-08-11T22:06:51.303089Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T21:10:35.151341Z","title":"InProceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, EMNLP 2023 - System Demonstrations, Singapore, December 6-10, 2023, pages 186–217","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.04947","last_updated":"2025-05-29T05:29:28Z","snapshot_observed_at":"2026-08-18T08:31:51.454475Z","submitted_at":"2024-12-06T11:07:44Z","title":"C$^2$LEVA: Toward Comprehensive and Contamination-Free Language Model Evaluation","version":3},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-11T21:10:35.151341Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.04947"},"observation_digest":"sha256:e877f893f1d5a39d5633a68eedd6ed4a0d694cc7f34f945278b690d137b82633","observation_id":"496bbc29-aa5d-4013-9cc9-e5810ee50300","resolution":{"observed_at":"2026-08-11T21:10:35.151341Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T18:03:57.173433Z","title":"Holistic evaluation of language models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.08291","last_updated":"2024-12-11T11:07:50Z","snapshot_observed_at":"2026-08-18T04:12:56.695006Z","submitted_at":"2024-12-11T11:07:50Z","title":"Code LLMs: A Taxonomy-based Survey","version":1},"reference_index":95,"source":"pdf_text","source_observed_at":"2026-08-11T18:03:57.173433Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.08291"},"observation_digest":"sha256:7c13457768182fffdad1f55d5f4c13f1af4cf61a6ffb08582abb7796459a2682","observation_id":"c29116bd-d715-461e-bc48-61ca2bbfff9a","resolution":{"observed_at":"2026-08-11T18:03:57.173433Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T18:03:22.759469Z","title":"BLIP-2: bootstrapping language- image pre-training with frozen image encoders and large language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.08307","last_updated":"2025-04-08T08:30:30Z","snapshot_observed_at":"2026-08-15T21:22:33.189437Z","submitted_at":"2024-12-11T11:39:42Z","title":"Investigating the Scaling Effect of Instruction Templates for Training Multimodal Language Model","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-11T18:03:22.759469Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.08307"},"observation_digest":"sha256:39c6cdd63e716fe258dcd7c8919279b1ad14728ca38316f7c907f064ec4ae99c","observation_id":"bd859e7b-4303-43e9-85be-3d014470063d","resolution":{"observed_at":"2026-08-11T18:03:22.759469Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T17:19:21.846142Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.09173","last_updated":"2024-12-12T11:03:25Z","snapshot_observed_at":"2026-08-20T13:36:20.102842Z","submitted_at":"2024-12-12T11:03:25Z","title":"ReFF: Reinforcing Format Faithfulness in Language Models across Varied Tasks","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-11T17:19:21.846142Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.09173"},"observation_digest":"sha256:0a1281b1d59e5ea4d25500df2b1f62ed2eaafb4dbed38c5c4095990b9023a030","observation_id":"84169d9f-473d-4b0e-ae81-120693f2d837","resolution":{"observed_at":"2026-08-11T17:19:21.846142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T15:02:33.258973Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.11414","last_updated":"2024-12-16T03:29:08Z","snapshot_observed_at":"2026-08-15T19:51:21.848714Z","submitted_at":"2024-12-16T03:29:08Z","title":"Biased or Flawed? Mitigating Stereotypes in Generative Language Models by Addressing Task-Specific Flaws","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-11T15:02:33.258973Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.11414"},"observation_digest":"sha256:687dd4e235f3a98cbb884932930a9b084a7dbb1cd9453dc70880bdfd19ecf9bb","observation_id":"3f8c4ce8-bd46-455b-b93b-5700c83e709a","resolution":{"observed_at":"2026-08-11T15:02:33.258973Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T17:29:02.773849Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.12170","last_updated":"2024-12-12T06:27:12Z","snapshot_observed_at":"2026-08-17T07:18:03.520995Z","submitted_at":"2024-12-12T06:27:12Z","title":"PickLLM: Context-Aware RL-Assisted Large Language Model Routing","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-11T17:29:02.773849Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.12170"},"observation_digest":"sha256:11b818212e520848524e9ea464e9a862d01364360d55ffecd1c592f2432d3fc1","observation_id":"add45c71-8f27-4ea8-87d5-bd84b9256717","resolution":{"observed_at":"2026-08-11T17:29:02.773849Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T12:58:26.892141Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.13666","last_updated":"2025-07-25T06:20:38Z","snapshot_observed_at":"2026-08-15T15:33:00.755397Z","submitted_at":"2024-12-18T09:48:53Z","title":"Evaluation of LLM Vulnerabilities to Being Misused for Personalized Disinformation Generation","version":2},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-11T12:58:26.892141Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.13666"},"observation_digest":"sha256:138203d740cda0ffb7866a5be6a6b80aa706ac78a699427b9561301c7456c10b","observation_id":"bcdbf6ba-05ba-469f-aa78-396ca4958c90","resolution":{"observed_at":"2026-08-11T12:58:26.892141Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T12:09:29.199485Z","title":"u ksekg \\","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.14574","last_updated":"2024-12-19T06:44:59Z","snapshot_observed_at":"2026-08-18T04:15:54.495842Z","submitted_at":"2024-12-19T06:44:59Z","title":"Sliding Windows Are Not the End: Exploring Full Ranking with Long-Context Large Language Models","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-11T12:09:29.199485Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.14574"},"observation_digest":"sha256:737684b5476f5c8f193a31fd398a180c8692c892eb30914bc8dc1d28df14f2bb","observation_id":"9f689af3-fa83-48a2-b8ad-cb04e59ac950","resolution":{"observed_at":"2026-08-11T12:09:29.199485Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T23:48:02.339502Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.01973","last_updated":"2025-01-09T07:26:05Z","snapshot_observed_at":"2026-08-18T10:45:51.182306Z","submitted_at":"2024-12-28T02:28:19Z","title":"INFELM: In-depth Fairness Evaluation of Large Text-To-Image Models","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-10T23:48:02.339502Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.01973"},"observation_digest":"sha256:27ec8098b773be24e12396d2dd12eda7c6a71f613d5c800b11cc8ac65de0e59f","observation_id":"aaf64c9c-8a63-44e6-abab-2941bf075d34","resolution":{"observed_at":"2026-08-10T23:48:02.339502Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T21:56:29.802738Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.03112","last_updated":"2025-01-06T16:20:44Z","snapshot_observed_at":"2026-08-15T03:59:43.337294Z","submitted_at":"2025-01-06T16:20:44Z","title":"LangFair: A Python Package for Assessing Bias and Fairness in Large Language Model Use Cases","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-10T21:56:29.802738Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.03112"},"observation_digest":"sha256:9fd2dba513c86d13473d03e95afa95fd60e16011c49c11b3ca4d521c05020507","observation_id":"f7e3189a-48ed-4eda-8eb0-f730cbd65023","resolution":{"observed_at":"2026-08-10T21:56:29.802738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-11T16:32:07.811499Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.04008","last_updated":"2025-03-16T21:12:51Z","snapshot_observed_at":"2026-08-17T17:36:51.974464Z","submitted_at":"2024-12-13T09:26:04Z","title":"A Generative AI-driven Metadata Modelling Approach","version":2},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-11T16:32:07.811499Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.04008"},"observation_digest":"sha256:352d90103b7ecab304ecf4dd305e43791e8c1b27bbf380345b2ac75b2b93b336","observation_id":"24d4414f-df42-44b7-8b4f-fe720994635c","resolution":{"observed_at":"2026-08-11T16:32:07.811499Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T22:17:56.116010Z","title":"Holistic Evaluation of Language Models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.04040","last_updated":"2025-02-09T08:00:36Z","snapshot_observed_at":"2026-08-15T09:18:34.265227Z","submitted_at":"2025-01-03T21:04:49Z","title":"A Survey on Large Language Models with some Insights on their Capabilities and Limitations","version":2},"reference_index":190,"source":"pdf_text","source_observed_at":"2026-08-10T22:17:56.116010Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.04040"},"observation_digest":"sha256:688c9d90b5f2b27e07ff05d574b8e7456107b840d6c97ead415fb4fa04badff4","observation_id":"11554f1f-f4e1-40dc-bc36-013fbacd976d","resolution":{"observed_at":"2026-08-10T22:17:56.116010Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T20:53:59.736259Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.07071","last_updated":"2025-06-02T15:40:42Z","snapshot_observed_at":"2026-08-18T12:47:51.466255Z","submitted_at":"2025-01-13T05:53:56Z","title":"Value Compass Benchmarks: A Platform for Fundamental and Validated Evaluation of LLMs Values","version":3},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-10T20:53:59.736259Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.07071"},"observation_digest":"sha256:0e0efce323cbad4403dfab20ae5e7946be641a19cad698df2741075ed5f69692","observation_id":"9d6439d2-e77b-4d61-a9ac-b786c99d9dec","resolution":{"observed_at":"2026-08-10T20:53:59.736259Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T20:34:34.841850Z","title":"Holistic Evaluation of Language Models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.08262","last_updated":"2025-01-14T17:21:16Z","snapshot_observed_at":"2026-08-14T19:33:53.600679Z","submitted_at":"2025-01-14T17:21:16Z","title":"Addressing the sustainable AI trilemma: a case study on LLM agents and RAG","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-10T20:34:34.841850Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.08262"},"observation_digest":"sha256:9faaa605a7ca28e665f91f013a55a3467f55e2bf6d3198738b20544803ab7a02","observation_id":"32469f6b-cb2d-49f3-b768-e031bb9cdc87","resolution":{"observed_at":"2026-08-10T20:34:34.841850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T15:39:33.230512Z","title":"Liang, R","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.13802","last_updated":"2025-03-09T16:39:06Z","snapshot_observed_at":"2026-08-15T08:30:40.334498Z","submitted_at":"2025-01-23T16:21:15Z","title":"Enhancing LLMs for Governance with Human Oversight: Evaluating and Aligning LLMs on Expert Classification of Climate Misinformation for Detecting False or Misleading Claims about Climate Change","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-10T15:39:33.230512Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2501.13802"},"observation_digest":"sha256:310493337d2dc95d35ce942f501fd28ed38f719fdf1b6fd2b4e7b6a0e0b7dcdf","observation_id":"9f47109e-10b6-477e-b1cf-8a0cc5bbacf4","resolution":{"observed_at":"2026-08-10T15:39:33.230512Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2502.00709","last_updated":"2026-04-16T23:58:55Z","snapshot_observed_at":"2026-08-16T17:42:11.418551Z","submitted_at":"2025-02-02T07:49:56Z","title":"RankFlow: A Multi-Role Collaborative Reranking Workflow Utilizing Large Language Models","version":5},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-05-23T04:36:32.897387Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.00709"},"observation_digest":"sha256:e680725475deb19ec0517f99b64ca1777f23e6e7722bbd7e7de888b4009b8a08","observation_id":"5d7cc790-2d6c-4439-912f-fb955a6bec0b","resolution":{"observed_at":"2026-05-23T04:37:31.933903Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-09T16:34:01.122641Z","title":"Narayanan, Yuhuai Wu, Ananya Kumar, Benjamin Newman, Binhang Yuan, Bobby Yan, Ce Zhang, Christian Cosgrove, Christopher D","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.01126","last_updated":"2025-02-03T07:43:27Z","snapshot_observed_at":"2026-08-14T04:22:06.342877Z","submitted_at":"2025-02-03T07:43:27Z","title":"Language Models Prefer What They Know: Relative Confidence Estimation via Confidence Preferences","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-09T16:34:01.122641Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.01126"},"observation_digest":"sha256:50038f76b25f3ce35d6263d4f07b435a37ba67c26ff39e02b14866fa52a0b420","observation_id":"06999197-03dc-4598-b2eb-405ce3fdcf68","resolution":{"observed_at":"2026-08-09T16:34:01.122641Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-09T11:22:28.913059Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.02743","last_updated":"2025-02-04T22:09:43Z","snapshot_observed_at":"2026-08-14T11:59:50.011836Z","submitted_at":"2025-02-04T22:09:43Z","title":"LLM Bandit: Cost-Efficient LLM Generation via Preference-Conditioned Dynamic Routing","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-09T11:22:28.913059Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.02743"},"observation_digest":"sha256:3c7adde32e0c18bd849fc5aa1c4fde929c077416a5e6519fb464ddbf3727a544","observation_id":"e6887e6a-0877-46d1-8f8a-f439804b5877","resolution":{"observed_at":"2026-08-09T11:22:28.913059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T20:51:52.861137Z","title":"Holistic evaluation of langu age models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.03472","last_updated":"2025-01-12T16:20:40Z","snapshot_observed_at":"2026-08-12T07:42:14.103912Z","submitted_at":"2025-01-12T16:20:40Z","title":"Powering LLM Regulation through Data: Bridging the Gap from Compute Thresholds to Customer Experiences","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-10T20:51:52.861137Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.03472"},"observation_digest":"sha256:d40ac156b106313ed72af9fd45789a7fa9b7175168d741b1cefc2ac8e546ffc2","observation_id":"706c8388-aaf6-4f52-ad7d-565da31aaef2","resolution":{"observed_at":"2026-08-10T20:51:52.861137Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-09T01:05:23.072022Z","title":"arXiv:2211.09110","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.03711","last_updated":"2025-02-06T01:58:48Z","snapshot_observed_at":"2026-08-16T10:40:27.468073Z","submitted_at":"2025-02-06T01:58:48Z","title":"MultiQ&A: An Analysis in Measuring Robustness via Automated Crowdsourcing of Question Perturbations and Answers","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-09T01:05:23.072022Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.03711"},"observation_digest":"sha256:f9cc2eb02366fb292b03c07303bc5bc46343bc85f23b7e1b8688616b37b23b00","observation_id":"651ef3b7-6b2e-4988-ad39-5486161d1280","resolution":{"observed_at":"2026-08-09T01:05:23.072022Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T15:06:55.036681Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06559","last_updated":"2025-05-25T23:36:47Z","snapshot_observed_at":"2026-08-19T05:02:22.089696Z","submitted_at":"2025-02-10T15:25:06Z","title":"Can We Trust AI Benchmarks? An Interdisciplinary Review of Current Issues in AI Evaluation","version":2},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-08T15:06:55.036681Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.06559"},"observation_digest":"sha256:cef13d3c0bdd2f09893fdf6d1bcf3c0fc5be7f0356d93975f76c5ef91ae8e582","observation_id":"f7e73f1a-be3b-4574-869d-ebff3a8d7aaa","resolution":{"observed_at":"2026-08-08T15:06:55.036681Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T14:51:15.426227Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.06655","last_updated":"2025-05-12T14:34:05Z","snapshot_observed_at":"2026-08-20T11:53:40.868172Z","submitted_at":"2025-02-10T16:45:18Z","title":"Unbiased Evaluation of Large Language Models from a Causal Perspective","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-08T14:51:15.426227Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.06655"},"observation_digest":"sha256:7a9dee46f6004d50e52daf7fab5254c9e24651378d929a6feeba91639d1ae891","observation_id":"eaeada13-0cc4-4762-9209-69f34cdc99f6","resolution":{"observed_at":"2026-08-08T14:51:15.426227Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T19:15:25.382826Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.06872","last_updated":"2025-02-08T06:50:47Z","snapshot_observed_at":"2026-08-16T08:10:55.902848Z","submitted_at":"2025-02-08T06:50:47Z","title":"Towards Trustworthy Retrieval Augmented Generation for Large Language Models: A Survey","version":1},"reference_index":104,"source":"pdf_text","source_observed_at":"2026-08-08T19:15:25.382826Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.06872"},"observation_digest":"sha256:62a79ffc47ac86f9d55b6531be0be723363c1b9ff832d5abb74de8b53bddb08c","observation_id":"e1aac824-a3ea-4db0-b811-de8ebc76936e","resolution":{"observed_at":"2026-08-08T19:15:25.382826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T13:10:56.377693Z","title":"arXiv:arXiv :2211.09110","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.07315","last_updated":"2025-02-23T09:44:00Z","snapshot_observed_at":"2026-08-17T12:21:52.355538Z","submitted_at":"2025-02-11T07:25:57Z","title":"White Hat Search Engine Optimization using Large Language Models","version":2},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-08T13:10:56.377693Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.07315"},"observation_digest":"sha256:54065d4b42a217e931eae0c91c339e1a1c53511d38cb0e17cc15ad8f138f8140","observation_id":"ce47f047-a3fb-4c34-a689-8de137f31e16","resolution":{"observed_at":"2026-08-08T13:10:56.377693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T11:44:59.733056Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.07747","last_updated":"2025-02-11T18:14:44Z","snapshot_observed_at":"2026-08-21T23:16:55.020188Z","submitted_at":"2025-02-11T18:14:44Z","title":"WHODUNIT: Evaluation benchmark for culprit detection in mystery stories","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-08T11:44:59.733056Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.07747"},"observation_digest":"sha256:5ab018662a0a96a516e967a9827c6308cffa51c5c42aef3762e7f56659482e2d","observation_id":"6ccb9e5d-77dc-4605-97b2-3ff4ae39264b","resolution":{"observed_at":"2026-08-08T11:44:59.733056Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T04:58:33.096413Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.08489","last_updated":"2025-02-13T17:33:24Z","snapshot_observed_at":"2026-08-13T23:11:01.152432Z","submitted_at":"2025-02-12T15:26:08Z","title":"Salamandra Technical Report","version":2},"reference_index":116,"source":"pdf_text","source_observed_at":"2026-08-08T04:58:33.096413Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.08489"},"observation_digest":"sha256:1d6623f037190c7f9e926f35af476314624aab068e95f47f5625798210449630","observation_id":"29d6a1a3-2b5b-4296-9a88-b5697a9071e9","resolution":{"observed_at":"2026-08-08T04:58:33.096413Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T04:54:50.768611Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.08512","last_updated":"2025-08-14T07:15:48Z","snapshot_observed_at":"2026-08-17T07:40:10.911779Z","submitted_at":"2025-02-12T15:46:34Z","title":"Measuring Diversity in Synthetic Datasets","version":3},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-08T04:54:50.768611Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.08512"},"observation_digest":"sha256:545476fabcf3b60c811a21100bfaf3e19efc9cf0763dcfa274353c672db1047f","observation_id":"08eb5c24-8535-4b84-8652-abfed5201ad3","resolution":{"observed_at":"2026-08-08T04:54:50.768611Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-07T23:35:42.752311Z","title":"Manning, Christo- pher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.09670","last_updated":"2025-02-12T22:55:43Z","snapshot_observed_at":"2026-08-14T13:00:35.954303Z","submitted_at":"2025-02-12T22:55:43Z","title":"The Science of Evaluating Foundation Models","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-07T23:35:42.752311Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.09670"},"observation_digest":"sha256:6ac35e874c34a99c8ea8ba6ee1338ffb927a1734117dab20ba5b272d9a440e70","observation_id":"88eb5930-81a1-40fd-bdd6-eebf5bcb7c84","resolution":{"observed_at":"2026-08-07T23:35:42.752311Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-07T19:19:38.204950Z","title":"In Proceedings of the International Confer- enceRecentAdvancesinNaturalLanguageProcess- ing,pages249–257,Hissar,Bulgaria.INCOMALtd","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2502.10140","last_updated":"2025-02-14T13:10:39Z","snapshot_observed_at":"2026-08-19T13:42:17.439496Z","submitted_at":"2025-02-14T13:10:39Z","title":"Small Models, Big Impact: Efficient Corpus and Graph-Based Adaptation of Small Multilingual Language Models for Low-Resource Languages","version":1},"reference_index":2015,"source":"pdf_text","source_observed_at":"2026-08-07T19:19:38.204950Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.10140"},"observation_digest":"sha256:cd64099fbfc7fa1eb51c813bb037544894b2dcc71d8dd16807071d718ac0affa","observation_id":"298370d4-fe6e-481a-ae59-4f4ed00a8fd4","resolution":{"observed_at":"2026-08-07T19:19:38.204950Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-10T10:07:33.771199Z","title":"”Holistic Evaluation of Language Models.” arXiv preprint arXiv:2211.09110 (2023)","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.15724","last_updated":"2025-01-28T11:34:22Z","snapshot_observed_at":"2026-08-18T10:17:36.796300Z","submitted_at":"2025-01-28T11:34:22Z","title":"Instruction-Based Fine-tuning of Open-Source LLMs for Predicting Customer Purchase Behaviors","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-10T10:07:33.771199Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.15724"},"observation_digest":"sha256:a149d2fb93eeebf9316bbb236063ba8d0ab0aa878a0f3090143aec66094b088d","observation_id":"53288b3d-354a-46b5-91bd-77643a3ce62d","resolution":{"observed_at":"2026-08-10T10:07:33.771199Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2502.18864","last_updated":"2025-02-26T06:17:13Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-02-26T06:17:13Z","title":"Towards an AI co-scientist","version":1},"reference_index":169,"source":"arxiv_source","source_observed_at":"2026-05-11T13:02:43.571234Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2502.18864"},"observation_digest":"sha256:b3c5de16018feb3707eeffc7f5b8c02083c76035178e566bd66c395ba423b97b","observation_id":"7004c954-9449-48ce-8b1e-ba9edb649fcd","resolution":{"observed_at":"2026-05-11T13:02:45.077542Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2503.16771","last_updated":"2026-04-12T13:23:28Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-03-21T01:00:45Z","title":"Enabling Global, Human-Centered Explanations for LLMs:From Tokens to Interpretable Code and Test Generation","version":3},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-05-22T23:41:22.017848Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2503.16771"},"observation_digest":"sha256:ab54f679fda84b1ada31898dd8b62d9d06c5884fbe25f10698142dbb8b9be675","observation_id":"f0bcf5e2-1a4f-4abd-9852-fb73ae166fe1","resolution":{"observed_at":"2026-05-22T23:42:16.115559Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T12:08:24.813142Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.13644","last_updated":"2025-04-18T11:50:30Z","snapshot_observed_at":"2026-08-19T10:19:40.599738Z","submitted_at":"2025-04-18T11:50:30Z","title":"Exploring the Potential for Large Language Models to Demonstrate Rational Probabilistic Beliefs","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-16T12:08:24.813142Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.13644"},"observation_digest":"sha256:e0af68c646d5dbc24b05752fcf624de71601890cadbabd34296ce5ff1b5c79a0","observation_id":"d2fbaf20-ec65-4cb3-b12b-59153d23f2eb","resolution":{"observed_at":"2026-08-16T12:08:24.813142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T11:39:10.918707Z","title":"Manning, Christopher R ´e, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.14985","last_updated":"2025-04-23T16:52:54Z","snapshot_observed_at":"2026-08-19T05:45:48.314083Z","submitted_at":"2025-04-21T09:26:05Z","title":"aiXamine: Simplified LLM Safety and Security","version":2},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-16T11:39:10.918707Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.14985"},"observation_digest":"sha256:0d5502b8f4eea2070844e4a54900f7f828ea5d22b179a096955f16e48b4100cf","observation_id":"d328c780-5f44-4b1a-9af2-d96949328338","resolution":{"observed_at":"2026-08-16T11:39:10.918707Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T12:04:24.264556Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.16116","last_updated":"2026-06-26T10:59:58Z","snapshot_observed_at":"2026-08-18T12:43:27.632940Z","submitted_at":"2025-04-18T16:40:39Z","title":"DMind Benchmark: Toward a Holistic Assessment of LLM Capabilities across the Web3 Domain","version":4},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-16T12:04:24.264556Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.16116"},"observation_digest":"sha256:0d71292b154540c728612f9b294e84d7cf81c712dc2410292a22980e6b2abf62","observation_id":"a93f9640-c6c5-4599-81fc-45e6f427374e","resolution":{"observed_at":"2026-08-16T12:04:24.264556Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2504.16155","last_updated":"2026-05-07T09:59:28Z","snapshot_observed_at":"2026-08-19T10:57:08.068728Z","submitted_at":"2025-04-22T17:52:04Z","title":"PRIMETIME : Limits of LLMs in Temporal Primitives","version":2},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-05-22T18:36:48.376877Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.16155"},"observation_digest":"sha256:10a0145c40517caa6ba62260ac1782120e5f16314b52619c17beea680b3c2fbc","observation_id":"17351bac-c664-4485-a0ae-67855d85dc35","resolution":{"observed_at":"2026-05-22T18:36:58.792994Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T10:21:26.005837Z","title":"arXiv:2211.09110 (Nov","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.18413","last_updated":"2025-04-25T15:14:25Z","snapshot_observed_at":"2026-08-18T16:11:20.779302Z","submitted_at":"2025-04-25T15:14:25Z","title":"An Empirical Study of Evaluating Long-form Question Answering","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-16T10:21:26.005837Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.18413"},"observation_digest":"sha256:2c6c3b659c435f35c6931e3095bccbf0a1bcf8ab6dc8f5390675100eb979da75","observation_id":"4e61d8c8-5d00-4acf-be30-a673f89df2b3","resolution":{"observed_at":"2026-08-16T10:21:26.005837Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T11:54:28.508916Z","title":"Holistic evaluation of language models.arXiv preprint arXiv:2211.09110, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18560","last_updated":"2025-04-19T16:18:22Z","snapshot_observed_at":"2026-08-22T17:55:09.494446Z","submitted_at":"2025-04-19T16:18:22Z","title":"Mind the Language Gap: Automated and Augmented Evaluation of Bias in LLMs for High- and Low-Resource Languages","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-16T11:54:28.508916Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.18560"},"observation_digest":"sha256:155fe5b5e3e6d90d5070c253dce799194647a47aa419a838683f8545f89a4683","observation_id":"12198f07-bd16-4b89-93ea-2ab43594bb64","resolution":{"observed_at":"2026-08-16T11:54:28.508916Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T05:24:24.018750Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.20771","last_updated":"2025-05-19T03:55:13Z","snapshot_observed_at":"2026-08-19T02:03:01.058892Z","submitted_at":"2025-04-29T13:52:47Z","title":"Computational Reasoning of Large Language Models","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-16T05:24:24.018750Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.20771"},"observation_digest":"sha256:dadc91d67404a808a2b0fed469ef47fa2449fddb60c89d87a5cbcf603b61c4c2","observation_id":"0ce24d1a-970a-41ea-a6bb-6f5aa27cf42e","resolution":{"observed_at":"2026-08-16T05:24:24.018750Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T11:00:03.339143Z","title":"arXiv preprint arXiv:2211.09110 (2022)","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.21027","last_updated":"2025-04-23T13:53:59Z","snapshot_observed_at":"2026-08-20T15:18:58.963603Z","submitted_at":"2025-04-23T13:53:59Z","title":"UrbanPlanBench: A Comprehensive Urban Planning Benchmark for Evaluating Large Language Models","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-16T11:00:03.339143Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2504.21027"},"observation_digest":"sha256:2b94c6cd10c81500b17005982644f7597fc0c2e1c4d392e486e5b88938161b76","observation_id":"1ad8c59c-a0d4-4d0b-8ca2-f856ac3b0ed8","resolution":{"observed_at":"2026-08-16T11:00:03.339143Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-16T04:37:48.205650Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.00853","last_updated":"2025-05-01T20:36:19Z","snapshot_observed_at":"2026-08-16T08:05:56.374210Z","submitted_at":"2025-05-01T20:36:19Z","title":"LLM Ethics Benchmark: A Three-Dimensional Assessment System for Evaluating Moral Reasoning in Large Language Models","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-16T04:37:48.205650Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.00853"},"observation_digest":"sha256:cb62f3dfda83d80de87f44735c9497c0f7e226462a4294d626e88f324072972c","observation_id":"831e7284-0cf6-45ee-9524-d6c5dd07aa13","resolution":{"observed_at":"2026-08-16T04:37:48.205650Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T23:58:41.842272Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.03475","last_updated":"2025-05-29T08:23:44Z","snapshot_observed_at":"2026-08-18T12:50:14.251276Z","submitted_at":"2025-05-06T12:28:50Z","title":"am-ELO: A Stable Framework for Arena-based LLM Evaluation","version":2},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-15T23:58:41.842272Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.03475"},"observation_digest":"sha256:b78a66409b32a63a551c8deff91feead99df2740073b15f9847e059c85546cf0","observation_id":"20c0818d-9ac5-454d-8572-b459a4629379","resolution":{"observed_at":"2026-08-15T23:58:41.842272Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T23:27:30.298577Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.04592","last_updated":"2025-05-07T17:35:36Z","snapshot_observed_at":"2026-08-17T23:42:59.217553Z","submitted_at":"2025-05-07T17:35:36Z","title":"AI Governance to Avoid Extinction: The Strategic Landscape and Actionable Research Questions","version":1},"reference_index":128,"source":"arxiv_source","source_observed_at":"2026-08-15T23:27:30.298577Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.04592"},"observation_digest":"sha256:59e7ff1aafff991bd134010f8a1611382c53f36c0b3b08cd0427754bb495c152","observation_id":"063e2f76-2474-4232-912d-b6090fcdedf5","resolution":{"observed_at":"2026-08-15T23:27:30.298577Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T23:23:39.355813Z","title":"”Holistic Eval- uation of Language Models.” arXiv preprint arXiv:2211.09110 (2023)","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.04806","last_updated":"2025-05-13T05:36:34Z","snapshot_observed_at":"2026-08-16T21:53:39.240862Z","submitted_at":"2025-05-07T21:15:40Z","title":"Red Teaming the Mind of the Machine: A Systematic Evaluation of Prompt Injection and Jailbreak Vulnerabilities in LLMs","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T23:23:39.355813Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.04806"},"observation_digest":"sha256:c911b3dee0ebf0722c028de46686f300b0c7e5b6613f53422523a0f5fe419d39","observation_id":"3184cf16-b833-4fc4-8ec3-ba3e28d3e9f5","resolution":{"observed_at":"2026-08-15T23:23:39.355813Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T23:06:00.739467Z","title":"Holistic Evaluation of Language Models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.05602","last_updated":"2025-07-13T12:48:23Z","snapshot_observed_at":"2026-08-22T05:54:10.557323Z","submitted_at":"2025-05-08T19:05:02Z","title":"HiBayES: A Hierarchical Bayesian Modeling Framework for AI Evaluation Statistics","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T23:06:00.739467Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.05602"},"observation_digest":"sha256:635005534478416c29c1de7e01d2d53a2b22eadb44df1f8059529d5845f18a39","observation_id":"71bb4ed7-d858-437c-b4b4-a8eece49a1ff","resolution":{"observed_at":"2026-08-15T23:06:00.739467Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T22:27:09.495968Z","title":"Liang, R","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.07233","last_updated":"2025-05-16T02:47:07Z","snapshot_observed_at":"2026-08-21T03:12:59.809550Z","submitted_at":"2025-05-12T05:19:01Z","title":"DynamicRAG: Leveraging Outputs of Large Language Model as Feedback for Dynamic Reranking in Retrieval-Augmented Generation","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T22:27:09.495968Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.07233"},"observation_digest":"sha256:b2508b007d759a1afefc82cdf5d49e72a9925e1da780b70efdd75b9aab0c4730","observation_id":"53e20df4-e36d-4458-8fff-d1076f7747c7","resolution":{"observed_at":"2026-08-15T22:27:09.495968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T22:26:12.477942Z","title":"Holistic evaluation of language models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.07239","last_updated":"2025-05-12T05:29:30Z","snapshot_observed_at":"2026-08-18T15:16:05.977819Z","submitted_at":"2025-05-12T05:29:30Z","title":"Comet: Accelerating Private Inference for Large Language Model by Predicting Activation Sparsity","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T22:26:12.477942Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.07239"},"observation_digest":"sha256:41c051fcaff172e6aa05eee0821fd0d9c59e8d2099005ed37d3ad979ee4fb34f","observation_id":"805b589e-0e5e-4e7c-9142-466157ef57ed","resolution":{"observed_at":"2026-08-15T22:26:12.477942Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T21:12:57.709230Z","title":", Bommasani, R","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.10472","last_updated":"2025-05-15T16:23:21Z","snapshot_observed_at":"2026-08-21T17:18:38.781127Z","submitted_at":"2025-05-15T16:23:21Z","title":"Large Language Models for Cancer Communication: Evaluating Linguistic Quality, Safety, and Accessibility in Generative AI","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-15T21:12:57.709230Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.10472"},"observation_digest":"sha256:8d31380b42ccbe80c3df49d0a4efa13b5551c8639e11b967fa3790f9d17293bc","observation_id":"0671a7e5-2dcd-46c1-80b4-dff2aade5ce8","resolution":{"observed_at":"2026-08-15T21:12:57.709230Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:58:22.855640Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.11365","last_updated":"2025-05-26T12:21:14Z","snapshot_observed_at":"2026-08-18T18:59:35.551356Z","submitted_at":"2025-05-16T15:31:08Z","title":"Phare: A Safety Probe for Large Language Models","version":4},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T20:58:22.855640Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.11365"},"observation_digest":"sha256:0ca8459ed4fea9b8328b4816a8a3d0798050b84b0f6087a1a9255c539ec67450","observation_id":"aef0beeb-d0b6-4e56-bc9a-aa0e8ae9a766","resolution":{"observed_at":"2026-08-15T20:58:22.855640Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:47:45.502670Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.12000","last_updated":"2025-05-17T13:24:08Z","snapshot_observed_at":"2026-08-18T20:20:45.683036Z","submitted_at":"2025-05-17T13:24:08Z","title":"IQBench: How \"Smart'' Are Vision-Language Models? A Study with Human IQ Tests","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T20:47:45.502670Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.12000"},"observation_digest":"sha256:c13806488041bb4d7ede970c4b6db3553991c7faf11221716799255622226da2","observation_id":"1e1b4266-b683-466f-8c1b-d68a8833ed8f","resolution":{"observed_at":"2026-08-15T20:47:45.502670Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:44:49.120283Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.12058","last_updated":"2025-05-17T15:40:03Z","snapshot_observed_at":"2026-08-19T10:29:31.277943Z","submitted_at":"2025-05-17T15:40:03Z","title":"Tiny QA Benchmark++: Ultra-Lightweight, Synthetic Multilingual Dataset Generation & Smoke-Tests for Continuous LLM Evaluation","version":1},"reference_index":1966,"source":"pdf_text","source_observed_at":"2026-08-15T20:44:49.120283Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.12058"},"observation_digest":"sha256:66f9b4ef708093187f0b032b485bd84aaf59fd2692f8988b48ef650c522f40b1","observation_id":"166db7f0-7f4d-4bf6-bdc2-f6e8ebf26969","resolution":{"observed_at":"2026-08-15T20:44:49.120283Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2505.12601","last_updated":"2026-05-14T18:08:42Z","snapshot_observed_at":"2026-08-15T04:53:18.316921Z","submitted_at":"2025-05-19T01:33:41Z","title":"Rethinking Predictive Modeling for LLM Routing: When Simple kNN Beats Complex Learned Routers","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-05-22T15:13:28.927880Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.12601"},"observation_digest":"sha256:856d5e350ee822a1f58f2d39cf56ea7e5e69c96663bae223741b69f0d1d5b809","observation_id":"7185838f-adde-4da3-89d5-65093b10910a","resolution":{"observed_at":"2026-05-22T15:14:57.484055Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:32:25.215708Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.12808","last_updated":"2025-05-19T07:34:25Z","snapshot_observed_at":"2026-08-19T00:12:06.194808Z","submitted_at":"2025-05-19T07:34:25Z","title":"Decentralized Arena: Towards Democratic and Scalable Automatic Evaluation of Language Models","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T20:32:25.215708Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.12808"},"observation_digest":"sha256:818841521cd41228732c23df187dd0c5c64a7724ce3b151dbe437b3e5e139fdc","observation_id":"6694cabc-06d1-4ad7-a6bc-433369601a04","resolution":{"observed_at":"2026-08-15T20:32:25.215708Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:22:29.929741Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.13098","last_updated":"2025-05-19T13:29:27Z","snapshot_observed_at":"2026-08-21T22:46:27.921383Z","submitted_at":"2025-05-19T13:29:27Z","title":"LLM-KG-Bench 3.0: A Compass for SemanticTechnology Capabilities in the Ocean of LLMs","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T20:22:29.929741Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.13098"},"observation_digest":"sha256:1758c73c840ea650759a310fe63d9baf623f7abf79b42bcc780e4dd534cc303d","observation_id":"3950c820-dd66-486e-bf5e-b65e403992db","resolution":{"observed_at":"2026-08-15T20:22:29.929741Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T20:18:59.931309Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.13291","last_updated":"2025-05-19T16:11:23Z","snapshot_observed_at":"2026-08-15T20:13:53.875819Z","submitted_at":"2025-05-19T16:11:23Z","title":"TimeSeriesGym: A Scalable Benchmark for (Time Series) Machine Learning Engineering Agents","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T20:18:59.931309Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.13291"},"observation_digest":"sha256:e86cc27fb38d813c5131eab4c07c3cc2110128e8092fb2e64a2717dc5301dde0","observation_id":"0b099e41-9d2c-41c1-abba-424777f6191b","resolution":{"observed_at":"2026-08-15T20:18:59.931309Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":"2211.09110","doi":"10.1007/bf01194075","metadata_source":"pith","pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Holistic Evaluation of Language Models","venue":"cs.CL","work_id":"cc02a01e-7218-47dc-8e66-3333e7e4adec","year":2022},"citing_paper":{"arxiv_id":"2505.15323","last_updated":"2026-04-03T14:23:00Z","snapshot_observed_at":"2026-08-14T23:42:59.581122Z","submitted_at":"2025-05-21T09:58:38Z","title":"Improving LLM First-Token Predictions in Multiple-Choice Question Answering via Output Prefilling","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-05-22T14:21:18.355360Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.15323"},"observation_digest":"sha256:b9e1f8ee7de4ceb5d13344cd6ad82573834a82c02d7a2e36369d1c3414ec3f7b","observation_id":"0f962621-e721-4d8a-8038-118360d4715b","resolution":{"observed_at":"2026-05-22T14:21:39.613433Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-07T15:12:13.198498Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.16128","last_updated":"2025-05-25T02:52:06Z","snapshot_observed_at":"2026-08-15T06:43:25.079629Z","submitted_at":"2025-05-22T02:13:48Z","title":"Veracity Bias and Beyond: Uncovering LLMs' Hidden Beliefs in Problem-Solving Reasoning","version":2},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-07T15:12:13.198498Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.16128"},"observation_digest":"sha256:6c347a387aec8912c63a7e92cce6151bc317b292ecba9c5e09040399d039aebf","observation_id":"0167f08b-7e13-4d3f-a7c7-7424af1f6053","resolution":{"observed_at":"2026-08-07T15:12:13.198498Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-07T15:07:00.526886Z","title":"Liang, R","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.16276","last_updated":"2025-05-22T06:21:40Z","snapshot_observed_at":"2026-08-22T03:28:09.243218Z","submitted_at":"2025-05-22T06:21:40Z","title":"How do Scaling Laws Apply to Knowledge Graph Engineering Tasks? The Impact of Model Size on Large Language Model Performance","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T15:07:00.526886Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.16276"},"observation_digest":"sha256:f57d3ab6f876bf58f1bdbdb90d7a4caab3c56a60d0dfa109ec5d6aa9bbca88ee","observation_id":"18a23f07-3b2a-4d2d-82e2-b3b24dd28cf7","resolution":{"observed_at":"2026-08-07T15:07:00.526886Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-07T15:12:22.022328Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.17131","last_updated":"2025-05-22T01:59:54Z","snapshot_observed_at":"2026-08-16T01:50:50.437369Z","submitted_at":"2025-05-22T01:59:54Z","title":"Relative Bias: A Comparative Framework for Quantifying Bias in LLMs","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-07T15:12:22.022328Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2505.17131"},"observation_digest":"sha256:b1ad5f5e406fed4c7a414cca5beeda92826b6896a2057e6a273fcf0cb0618e06","observation_id":"9e817d53-75a9-4742-972f-1035cb756a88","resolution":{"observed_at":"2026-08-07T15:12:22.022328Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2211.09110/citation-record","integrity":"/paper/2211.09110/integrity","json":"/paper/2211.09110/citation-record.json","paper":"/paper/2211.09110"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2005.14165","last_updated":"2020-07-22T19:47:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-28T17:29:03Z","title":"Language Models are Few-Shot Learners","version":4},"cited_work":{"arxiv_id":"2005.14165","doi":"10.1145/1926385.1926423","metadata_source":"pith","pith_arxiv_id":"2005.14165","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Language Models are Few-Shot Learners","venue":"cs.CL","work_id":"214732c0-2edd-44a0-af9e-28184a2b8279","year":2020},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/2005.14165","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:0f20eb9ea534cb3dc68f9bec188eba032679d0da8810328a3cf6a4733c2b5c1c","observation_id":"fd2b9ddb-6b51-4c64-b546-8f994c8bb60e","resolution":{"observed_at":"2026-05-24T10:09:19.266423Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-07-09T08:48:33.42086+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-09T08:48:33.42086+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.acl-long.150","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"doi: 10.18653/v1/2021.acl-long.150","venue":null,"work_id":"28ca0026-3906-4281-9006-088b556137d2","year":2021},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:9996f408e7c83163970b20fda6ce37c804966da935254565cb6ebb99672861cd","observation_id":"4fc7afca-6d7e-46e3-af1d-835c2d1bc741","resolution":{"observed_at":"2026-05-24T10:09:19.299324Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.5281/zenodo.4761960","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"URLhttps://glottolog.org/accessed2021-08-08","venue":"Zenodo (CERN European Organization for Nuclear Research)","work_id":"b03a94d5-21f1-4c7f-8e70-54e70f8cd4db","year":2018},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:75ff6db5b8f521eefd2ec5f4b1edb0fd92a506fb7ff852e67d026a32a3f0501c","observation_id":"d1b20050-8ea6-4d4b-ba5b-c3abdc75b098","resolution":{"observed_at":"2026-05-24T10:09:19.283766Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-14T15:19:05.440904Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":"2105.09938","doi":"10.18653/v1/2021.eacl-main.225","metadata_source":"pith","pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Measuring Coding Challenge Competence With APPS","venue":"cs.SE","work_id":"c014c12f-1080-4cb2-ae03-ab6b7c09445c","year":2021},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:ae5994753790ac360d7f1175b8bc8af079e2ae5394a0b51655fda6b7f9cf3059","observation_id":"7b7f0655-19dc-4471-b9be-85b3bb247e18","resolution":{"observed_at":"2026-05-24T10:09:19.289641Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"oxfordhb/9780199","doi":"10.1093/oxfordhb/9780199286546.001.0001/","metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-11T11:50:26.030339Z","title":"In Christopher Hitchcock & Alan Hajek, edi- tors: Oxford Handbook of Probability and Philosophy , Oxford University Press, pp","venue":null,"work_id":"14395009-699b-40c7-94de-6004ef131037","year":2008},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:87fdcecd0f5c0561290eea418297ced8dd2d4aafd51beec4e8c272bf9631cd78","observation_id":"8e60302d-56ca-4737-8cc1-67f5948ec5e3","resolution":{"observed_at":"2026-05-24T10:09:19.254183Z","resolver_source":"arxiv_id","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1016/j.cognition.2007.05.006","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Cognition , year =","venue":"Cognition","work_id":"b71d974a-c34c-4e5e-8062-c7e8770974ba","year":2008},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:c7d1df0a2696c0a9544fa7166736a71de0bb36a1e7b030cc1c82bfac11ab3f28","observation_id":"333e3eee-7498-4e14-a12c-24d304c33015","resolution":{"observed_at":"2026-05-24T10:09:19.273599Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-17T21:08:19.06881+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-17T21:08:19.06881+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1806.08730","last_updated":"2018-06-20T16:39:26Z","snapshot_observed_at":"2026-08-17T06:16:47.122449Z","submitted_at":"2018-06-20T16:39:26Z","title":"The Natural Language Decathlon: Multitask Learning as Question Answering","version":1},"cited_work":{"arxiv_id":"1806.08730","doi":"10.2466/pr0.1957.3.3.635","metadata_source":"pith","pith_arxiv_id":"1806.08730","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"The Natural Language Decathlon: Multitask Learning as Question Answering","venue":"cs.CL","work_id":"75b95963-bb63-4588-816d-17e4bf36092e","year":2018},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/1806.08730","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:a75faade5e9b2bc1280188190e90d0334f1d6cf9264adc4800ee40f4a6cedebe","observation_id":"655b68ab-0010-4a0d-9fbf-3528748e4909","resolution":{"observed_at":"2026-05-24T10:09:19.279889Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2202.03286","last_updated":"2022-02-07T15:22:17Z","snapshot_observed_at":"2026-08-15T19:46:54.909325Z","submitted_at":"2022-02-07T15:22:17Z","title":"Red Teaming Language Models with Language Models","version":1},"cited_work":{"arxiv_id":"2202.03286","doi":"10.48550/arxiv.2202.03286","metadata_source":"pith","pith_arxiv_id":"2202.03286","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Red Teaming Language Models with Language Models","venue":"cs.CL","work_id":"d1274c54-508f-42f9-aeb3-91db13f3a622","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"cited_paper":"/paper/2202.03286","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:4556861b986958b8f60d9191e5a1a7e383815e6b760cf205345a2d6ffaf393d3","observation_id":"c7150b94-318f-4e30-8f44-5d369f022534","resolution":{"observed_at":"2026-05-24T10:09:19.693997Z","resolver_source":"local_arxiv","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-14T22:08:09.310411+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-14T22:08:09.310411+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.03350","last_updated":"2023-10-17T18:57:17Z","snapshot_observed_at":"2026-08-21T16:05:02.584580Z","submitted_at":"2022-10-07T06:50:23Z","title":"Measuring and Narrowing the Compositionality Gap in Language Models","version":3},"cited_work":{"arxiv_id":"2210.03350","doi":"10.1007/s11229-022-03791-y","metadata_source":"pith","pith_arxiv_id":"2210.03350","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Measuring and Narrowing the Compositionality Gap in Language Models","venue":"cs.CL","work_id":"79aa4add-ff4a-4871-9c74-6f473b0579c1","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-11T11:50:26.030339Z"},"links":{"cited_paper":"/paper/2210.03350","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:a73c5d5316ae07ff8b71305b239effa82f5390a875d3fe90aa7d12a24a424efa","observation_id":"c7b09df6-a83c-4bda-affe-0aebbf696222","resolution":{"observed_at":"2026-05-24T10:09:19.294997Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.05100","last_updated":"2023-06-27T09:57:58Z","snapshot_observed_at":"2026-08-04T18:56:03.233715Z","submitted_at":"2022-11-09T18:48:09Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","version":4},"cited_work":{"arxiv_id":"2211.05100","doi":"10.48550/arxiv.2211.05100","metadata_source":"pith","pith_arxiv_id":"2211.05100","snapshot_observed_at":"2026-07-10T06:15:00.866473Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","venue":"cs.CL","work_id":"337ba690-f35d-4154-9450-8edf4bc9f488","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"cited_paper":"/paper/2211.05100","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:a44be670ba881de42f47af3ec6ef77a2afcaf2172aa6686812d88aa390951ad4","observation_id":"02724bfe-2e4b-42b2-beaf-c73880838e9c","resolution":{"observed_at":"2026-05-24T10:09:19.247836Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-25T16:25:57.975024+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-25T16:25:57.975024+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0276.246027","doi":"10.1145/2460276.2460278","metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Yes” or “No","venue":"Queue","work_id":"91965f25-2acc-4ae2-b3f1-96a5347551cd","year":2014},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:979ca91767d1e7037bbdd5f1dc62ba0c5bf3ee9ff83ab1b0496bae1859a98c11","observation_id":"bbe34219-0bfc-444e-9d46-7e8bfdb9d9c2","resolution":{"observed_at":"2026-05-24T10:09:19.260372Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"8265190e-93ac-495f-a516-9d904041cf9c","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:3f4365fa01770ee0dbbda7672b67a710bf55044e9dbe34b8593615bf327fde71","observation_id":"73b5c53d-ac5d-40d7-80f5-ae782b89d9e2","resolution":{"observed_at":"2026-05-24T10:19:19.794025Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"d9607ab4-1e5f-4fc9-ba2b-464681f9db3b","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:08955648b8a327f40c7de03246a613ce49791b1dae8eb7f15c63ca4be6c7de02","observation_id":"99063e66-2f6c-4e86-9ca2-88dc4e348cd1","resolution":{"observed_at":"2026-05-24T10:19:19.797403Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Grandfather","venue":null,"work_id":"cd66469d-081a-49fc-8eb0-3f973d7db6a6","year":2018},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:f0140a131e71999926833ba3b9d72708c1831cf2add36b21362ae59f8eb99aa1","observation_id":"d10ca844-e51b-418c-b639-d9e2b9f95339","resolution":{"observed_at":"2026-05-24T10:19:19.813945Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"(2017), which derives its list form Greenwald et al","venue":null,"work_id":"9e90edaf-e1b8-40bb-94a2-adba8b39297b","year":2017},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:dadf6bedabace08008e0ed5f43747244099091e0eb895563679843b35d0999dc","observation_id":"85ae1069-f0e0-4dac-9230-449b76a51acf","resolution":{"observed_at":"2026-05-24T10:19:19.805151Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"(2018), which derives its list form Chalabi & Flowers (2017)","venue":null,"work_id":"dbc3dfa0-d561-470e-82e2-9df92394cee0","year":2018},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:bec32cc6c63394b2fd20712cf57f7488cf108b84585665d715a243db07633b8e","observation_id":"edd232ba-c256-498d-8d6d-8b31e9d535ce","resolution":{"observed_at":"2026-05-24T10:19:19.811250Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"It came from down here","venue":null,"work_id":"57847938-6283-4a51-8c1e-85cb93c2447f","year":2020},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:dc0f94e3b47c1c7d79f9ae34c7b9357422e32db3d3a93f64acb7cf43b92b272e","observation_id":"20e4603d-6d17-4152-bf89-71769e843b82","resolution":{"observed_at":"2026-05-24T10:19:19.802573Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"42c171dc-b33f-42f1-87ac-170ae5620cfb","year":null},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:e7d57a99cce1faef6d2ec85c5976fbcee331732badf67072c67600f4fa45e8cb","observation_id":"740f62a9-1862-4a24-bf73-b9fb2a035933","resolution":{"observed_at":"2026-05-24T10:19:19.800328Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"1a073121-6b70-45f4-ad80-8f72d6ab2bfe","year":2023},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:a4766f102c6868ac7b8d9382ea161895dbd55617558fc71576232af8adca44fe","observation_id":"ce64fb07-b7be-4c50-8c71-8c8963c0a340","resolution":{"observed_at":"2026-05-24T10:19:19.807714Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"The capital of France is __","venue":null,"work_id":"e8555b6b-3f1a-4742-81b8-c0b2bd547845","year":2046},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:b4a074e28167a02424f76ea5338c46a3598ceb7e4bb83c541fff391c8e20bc63","observation_id":"37e2122b-5091-448f-890e-09361fe234a0","resolution":{"observed_at":"2026-05-24T10:19:19.791793Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.05100","last_updated":"2023-06-27T09:57:58Z","snapshot_observed_at":"2026-08-04T18:56:03.233715Z","submitted_at":"2022-11-09T18:48:09Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","version":4},"cited_work":{"arxiv_id":"2211.05100","doi":"10.48550/arxiv.2211.05100","metadata_source":"pith","pith_arxiv_id":"2211.05100","snapshot_observed_at":"2026-07-10T06:15:00.866473Z","title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","venue":"cs.CL","work_id":"337ba690-f35d-4154-9450-8edf4bc9f488","year":2022},"citing_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-05-24T10:06:30.552785Z"},"links":{"cited_paper":"/paper/2211.05100","citing_paper":"/paper/2211.09110"},"observation_digest":"sha256:1a79de77588d522453d3b0d7a145baead7c68b6ec86d2bc2f598cfae1a5e13e8","observation_id":"95af2df3-d032-4c58-b63e-9a27e2ff1c79","resolution":{"observed_at":"2026-05-24T10:09:19.667240Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-25T16:25:57.975024+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-25T16:25:57.975024+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models"},"reference_resolution":{"displayed":21,"state_counts":{"malformed_identifier":2,"metadata_mismatch":7,"parse_uncertain":0,"unresolved":4,"verified_exact":3,"verified_fuzzy":5},"total_outbound_references":21},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"thesis":"As of 23 August 2026, this Paper Citation Record lists 21 of 21 outbound references and 100 inbound Pith citation observations for arXiv:2211.09110."}