{"as_of":"2026-08-09T02:06:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:1374ffa983886828aad1addea100ce035941668850219c910043c3a172ec68e2","coverage":[{"denominator":51,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":51,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T10:31:02.319589Z","state":"measured"},{"denominator":59,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":59,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":8,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":8,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T13:46:08.844007Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-30T23:45:07.998622Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2512.10687","last_updated":"2026-04-20T11:46:57Z","snapshot_observed_at":"2026-08-02T16:01:53.254884Z","submitted_at":"2025-12-11T14:34:40Z","title":"Safe for Whom? Rethinking How We Evaluate the Safety of LLMs for Real Users","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-05-16T23:22:42.431997Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2512.10687"},"observation_digest":"sha256:2d8972fec1fe9be8948ff811d7227c7cdf9b0121de597e0d3b1c74b0291b55ed","observation_id":"cd860ed7-1571-4add-9e0d-9db5135b3653","resolution":{"observed_at":"2026-05-16T23:23:39.942350Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2604.26500","last_updated":"2026-07-03T09:45:31Z","snapshot_observed_at":"2026-07-31T15:15:29.527162Z","submitted_at":"2026-04-29T10:03:12Z","title":"StarDrinks: An English and Korean Test Set for SLU Evaluation in a Drink Ordering Scenario","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-05-07T11:27:03.620795Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2604.26500"},"observation_digest":"sha256:19e3f5b7508dbccdb1d36820685a7ff6ae39893ccb8fe8549ed846cc27d9f55c","observation_id":"a7bd56d7-f80a-4bfa-be9e-c15e06bc3b09","resolution":{"observed_at":"2026-05-12T09:21:25.388199Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.07053","last_updated":"2026-05-26T07:47:31Z","snapshot_observed_at":"2026-08-02T22:34:50.237073Z","submitted_at":"2026-05-08T00:02:39Z","title":"GSM-SEM: Benchmark and Framework for Generating Semantically Variant Augmentations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-05-11T00:57:55.616036Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.07053"},"observation_digest":"sha256:1d26dbe74a5d2980d07103399ae919ba86fa81271c6d235223a135b425075d21","observation_id":"b9447640-1d57-4f04-bb6a-4e4ce0214ec9","resolution":{"observed_at":"2026-05-11T04:55:59.932740Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.07053","last_updated":"2026-05-26T07:47:31Z","snapshot_observed_at":"2026-08-02T22:34:50.237073Z","submitted_at":"2026-05-08T00:02:39Z","title":"GSM-SEM: Benchmark and Framework for Generating Semantically Variant Augmentations","version":2},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-06-30T23:42:00.965554Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.07053"},"observation_digest":"sha256:8a8a04a5397e7e9ce79af7b2c5a8d39b4e7f09fca0c96e71fb861c3589038c54","observation_id":"86bb03c9-9819-40e2-95cb-6390ff59ae8e","resolution":{"observed_at":"2026-06-30T23:45:08.000489Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.19382","last_updated":"2026-05-19T05:28:54Z","snapshot_observed_at":"2026-07-06T23:30:06.876413Z","submitted_at":"2026-05-19T05:28:54Z","title":"PRISM: A Benchmark for Programmatic Spatial-Temporal Reasoning","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-05-20T05:52:46.575173Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.19382"},"observation_digest":"sha256:0e2b86384d4f6ca64b1ad2000355a4d0353430dcd8e2ea304abe43026fb01667","observation_id":"8e54f074-d4a3-4308-8237-8717c9608525","resolution":{"observed_at":"2026-05-20T05:53:04.325279Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.27209","last_updated":"2026-05-26T16:02:00Z","snapshot_observed_at":"2026-08-03T11:00:10.339186Z","submitted_at":"2026-05-26T16:02:00Z","title":"Learning to Act under Noise: Enhancing Agent Robustness via Noisy Environments","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-06-29T16:51:36.524194Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.27209"},"observation_digest":"sha256:e6bfba565060f63a7505c3ae9a8012af5ad9e8493c989df571b81f96ce0655cf","observation_id":"b0a32282-5a01-4201-ade7-b59c79931a23","resolution":{"observed_at":"2026-06-29T16:53:40.568253Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.30738","last_updated":"2026-05-29T02:02:31Z","snapshot_observed_at":"2026-08-01T17:11:51.059319Z","submitted_at":"2026-05-29T02:02:31Z","title":"MAVEN: Improving Generalization in Agentic Tool Calling","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-06-28T22:42:11.459778Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.30738"},"observation_digest":"sha256:6fbb3796e9801b66de7430233f8bdad1d04082621a9896b6bd0f0aee7cf582de","observation_id":"cacb1980-62da-4e63-b519-bb6e4d88023a","resolution":{"observed_at":"2026-06-28T22:42:46.135066Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-08-02T13:46:08.844007Z","title":"Lunardi, V","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-08T05:06:39.163957Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.844007Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:76d986d6c899131467b227044c1d707fe7265bbac0cec553b3b2efc49451e8b9","observation_id":"b9caf502-53cd-465a-a0b2-1d1934c03339","resolution":{"observed_at":"2026-08-02T13:46:08.844007Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2509.04013/citation-record","integrity":"/paper/2509.04013/integrity","json":"/paper/2509.04013/citation-record.json","paper":"/paper/2509.04013"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-05T10:31:02.079925Z","title":"Austin et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.079925Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:30f4122cbead1213fb3539bb95aca32e98951ae957165634a060a3bd575d1876","observation_id":"128e5b78-77f7-44b3-8ba5-540d8af3ab63","resolution":{"observed_at":"2026-08-05T10:31:02.079925Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14762","last_updated":"2024-11-05T16:40:21Z","snapshot_observed_at":"2026-07-06T17:34:07.737296Z","submitted_at":"2024-02-22T18:21:59Z","title":"MT-Bench-101: A Fine-Grained Benchmark for Evaluating Large Language Models in Multi-Turn Dialogues","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14762","snapshot_observed_at":"2026-08-05T10:31:02.085778Z","title":"Bai et al","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.085778Z"},"links":{"cited_paper":"/paper/2402.14762","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:e6d692950fb6849cc65eb353c82971361e0a8bfcb51cbebbe8cb4df8db989c9f","observation_id":"032f5bc1-5851-425b-9f95-bae93273db73","resolution":{"observed_at":"2026-08-05T10:31:02.085778Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.276805Z","title":"Bailey, N","venue":null,"work_id":"4f35a8ad-fcd3-4067-91cb-cacab403ec62","year":2008},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.090860Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:1d5773c7bf43bef7456d6a1b562d646a9d28fb2c47b14fbfb35aff7c35fe8ff8","observation_id":"5e28eac2-5474-4fa9-9b64-922e1d3182a5","resolution":{"observed_at":"2026-08-05T10:31:03.281185Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.262796Z","title":null,"venue":null,"work_id":"571c5bc0-1114-416c-b411-418b8d82174c","year":2020},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.095785Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:18fc16289840eabf0daeb6b4b4c414e3532fd42aee524c33b136d684919eec00","observation_id":"766cc208-33f1-4b9c-adb1-e312d203b2bd","resolution":{"observed_at":"2026-08-05T10:31:03.267003Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.248066Z","title":"Burnell et al","venue":null,"work_id":"fd6d0acf-4966-476f-9f5a-bcb7dd8bd3f8","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.101619Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:4efbd9fa3026503ae4f52de49207138ab3215f0dea998956b5a04ab816e78632","observation_id":"a3b2f874-c6b9-4767-9bdd-7e6b6e164b3a","resolution":{"observed_at":"2026-08-05T10:31:03.252544Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.232995Z","title":"Carterette, J","venue":null,"work_id":"0ef1ebd9-366a-4d71-b9ba-b10704ac9a99","year":2006},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.106197Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:2ef976dca9902099108e76c13c942c240af7bb3f24ccf8c6bc72825d4b9c61f9","observation_id":"5c5a3774-d159-438d-8764-c88fab8a9d99","resolution":{"observed_at":"2026-08-05T10:31:03.237912Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.219213Z","title":"Carterette, A","venue":null,"work_id":"829cffc5-792a-4868-9f8c-33976d533609","year":2015},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.111083Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:a27bcd0373576a01eb63482a48bb6bf83606bc87f5f71148c564132c047435c0","observation_id":"555fd79b-d022-455b-9f0d-cdfb09ccc4a0","resolution":{"observed_at":"2026-08-05T10:31:03.223737Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-05T10:31:02.115716Z","title":"Chen et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.115716Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:908d84eb8252559bd22b992ceea64b63931df0180c8674669f436dd9d60ee4f4","observation_id":"0952470b-d78b-4187-b02c-03e45a157225","resolution":{"observed_at":"2026-08-05T10:31:02.115716Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.204980Z","title":null,"venue":null,"work_id":"fe7ddb9b-5dab-4009-afe7-cbfad782dd1e","year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.120756Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:0a9181190abe603fa3cb7c13f4e17f25be7f5e942cd2ea39e75624e404027a91","observation_id":"73abcfd4-d6c3-4275-beb0-e0e23c633649","resolution":{"observed_at":"2026-08-05T10:31:03.209510Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.189835Z","title":"Clark, K","venue":null,"work_id":"a95ab14e-884e-4d51-a24d-29af50e180c3","year":2019},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.125172Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:82851c0cd34308d0cf10ba81367e1e8a7dc9cf935e6005824c722adf033737b7","observation_id":"aa1b7741-f64a-4045-92a0-ab886997637b","resolution":{"observed_at":"2026-08-05T10:31:03.194167Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-05T10:31:02.129784Z","title":"Clark, I","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.129784Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:a0bca55ea9b995b1d7b2d12f47a180f21d80a939821ab989cfcd3a5c0e316fe2","observation_id":"cad664e0-b2a2-46bc-b0f5-99639c46deb1","resolution":{"observed_at":"2026-08-05T10:31:02.129784Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-05T10:31:02.134983Z","title":"Cobbe, V","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.134983Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:c50911a154057bbee51b99343dd26dbd5160ae8115078a6adc97d28ff0f8e74f","observation_id":"97370bb6-f807-4ab7-86d1-458051a8a23f","resolution":{"observed_at":"2026-08-05T10:31:02.134983Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.175680Z","title":"Frohberg and F","venue":null,"work_id":"a16e988b-d0b5-40da-84fc-b7a9a4efe273","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.140039Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:98c3e453a16dc082ad81eefdca99b3ad85bfd3d21675a35b390076684e9a0ebd","observation_id":"3e54834d-57eb-4a74-9042-3cf4dbc2279b","resolution":{"observed_at":"2026-08-05T10:31:03.180081Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.161913Z","title":"Guiver, S","venue":null,"work_id":"8fc0995c-a3ba-4d62-a31b-42703f699841","year":2009},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.144420Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:e9aaad70a62ca560fbbbeb9ed07a72ba72c8fe86d5fda18d484e3d3b4dd931b5","observation_id":"2b89a67e-665c-4c54-b7c8-78dc26687eee","resolution":{"observed_at":"2026-08-05T10:31:03.166258Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.147852Z","title":null,"venue":null,"work_id":"9e2bc9e6-e006-45d1-ae11-5b6f58a6bd6a","year":2011},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.149440Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:f4dcec6e07dceed6bce2c5320093c55016064cd9e71fe309409c8d2048289dd8","observation_id":"1ae1f1f2-014a-4992-96a9-b8f3e2c3a9ab","resolution":{"observed_at":"2026-08-05T10:31:03.152235Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.133358Z","title":"Hendrycks, C","venue":null,"work_id":"2520f4e3-30e7-41cc-8e4f-22baab2cba60","year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.153996Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:b0b8661734f8c43f0c2834c2aa5b04b70398e5a01867746306414a952fbbef35","observation_id":"094ceee2-e613-465d-9876-fff9a937eda6","resolution":{"observed_at":"2026-08-05T10:31:03.137847Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.03874","last_updated":"2021-11-08T21:30:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-03-05T18:59:39Z","title":"Measuring Mathematical Problem Solving With the MATH Dataset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.03874","snapshot_observed_at":"2026-08-05T10:31:02.158497Z","title":"Hendrycks et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.158497Z"},"links":{"cited_paper":"/paper/2103.03874","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:91e9590e6be8c6fbfd0879a84556a79f79e3270cf1897ddf932993f135989138","observation_id":"04a26570-1182-4b11-bdd9-7fe9326ba056","resolution":{"observed_at":"2026-08-05T10:31:02.158497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.120027Z","title":"Kim et al","venue":null,"work_id":"b1e92372-6eee-49ba-96a5-2428841a9627","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.163069Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:4f6fa1591a56bea3d1544154e02a83c6f786bffecdd73fed049d9bb5fe018c9e","observation_id":"1701abe3-569f-4e95-ba65-170ea1c8c103","resolution":{"observed_at":"2026-08-05T10:31:03.124043Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.106132Z","title":"Kojima, S","venue":null,"work_id":"a5af94bb-a515-444d-bf3c-a70a7b7d7de0","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.167804Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:ae0df3357fea6cce1c787b768c3e732bd28dab1bd0465bf63fe2afe392396d3d","observation_id":"f47fa918-c496-460b-bb29-1aa8ff18da21","resolution":{"observed_at":"2026-08-05T10:31:03.110964Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.091645Z","title":null,"venue":null,"work_id":"7f7f5d95-d679-407b-8bfe-81e1f56d8fef","year":2017},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.172823Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:beaa8523869b90c8f5f02a0cd4803c530433a6a7db5bcc905ed023aa7cf6579e","observation_id":"d0e77fad-3de6-4f99-b2c2-a5e45e072299","resolution":{"observed_at":"2026-08-05T10:31:03.096351Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.14215","last_updated":"2024-11-21T15:25:08Z","snapshot_observed_at":"2026-07-06T19:53:47.605701Z","submitted_at":"2024-11-21T15:25:08Z","title":"Evaluating the Robustness of Analogical Reasoning in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.14215","snapshot_observed_at":"2026-08-05T10:31:02.177268Z","title":"Lewis and M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.177268Z"},"links":{"cited_paper":"/paper/2411.14215","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:95d09b846e22bc38da2711c69b63d03f801d0d8e92c79a113a515f06271dd751","observation_id":"25a96312-2b0b-475c-91b2-649f60b911e9","resolution":{"observed_at":"2026-08-05T10:31:02.177268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2102.04664","last_updated":"2021-03-16T08:28:37Z","snapshot_observed_at":"2026-07-06T10:39:42.676631Z","submitted_at":"2021-02-09T06:16:25Z","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2102.04664","snapshot_observed_at":"2026-08-05T10:31:02.182126Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.182126Z"},"links":{"cited_paper":"/paper/2102.04664","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:5336d5c7f4f4409ad119c5dde47b636a8644e3a703ae102b5292321832ffa2da","observation_id":"887ea66b-3234-48da-a262-04da90ae8d1f","resolution":{"observed_at":"2026-08-05T10:31:02.182126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.076677Z","title":"Lunardi, D","venue":null,"work_id":"66268753-c069-4deb-bc73-8505a049c38d","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.186580Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:686808ed01d63a429d872c699a3780dac6cece07e387830d7e81e0508c170ae7","observation_id":"c0f0c23a-571e-4cbd-904c-acbbbaa255fd","resolution":{"observed_at":"2026-08-05T10:31:03.081688Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.062203Z","title":"Mihaylov, P","venue":null,"work_id":"2fac6036-8c29-445d-8839-c3e3e1a1be7a","year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.191311Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:742f691dce2dcd7828109b53b772d78e3467526834c9799a42eb17051df310cb","observation_id":"96ccef58-4bca-4a9b-808f-ac8f72575dd8","resolution":{"observed_at":"2026-08-05T10:31:03.066683Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.047930Z","title":"Mitchell","venue":null,"work_id":"efab1f26-62b0-468b-9cdc-0efecfe3a8a8","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.196651Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:dd9b4cc483afa73c92c620190a2c8fa93152423b357e0d4802597c60608353db","observation_id":"b1d0c527-9588-48b4-8bb9-4dbb8515e46e","resolution":{"observed_at":"2026-08-05T10:31:03.052407Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.034236Z","title":"Mitchell","venue":null,"work_id":"3f26a3f2-0dcd-47d9-bc38-8113fb7a7681","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.201503Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:a65388f4f5378f9e613563c2cb81f091ff86f9d91ef6efc43d475cc3547bee6e","observation_id":"226d59b7-7fb1-4c96-9e39-54c25d67787e","resolution":{"observed_at":"2026-08-05T10:31:03.038500Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1611.09268","last_updated":"2018-10-31T14:46:47Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2016-11-28T18:14:11Z","title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1611.09268","snapshot_observed_at":"2026-08-05T10:31:02.206347Z","title":"Nguyen, M","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.206347Z"},"links":{"cited_paper":"/paper/1611.09268","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:0df8e9963bc56da6a2b62b41235f93ac9eeac739954cbacae99a04d7634acde8","observation_id":"33a6f282-cfff-4201-bc73-309166eb850f","resolution":{"observed_at":"2026-08-05T10:31:02.206347Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.019911Z","title":"Ouyang, J","venue":null,"work_id":"32606852-2b24-434d-9cdc-2ebeed621c90","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.211401Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:714d84d811f53bfaed120d983c92b2e992566c0b5f39e7502e9debf639962164","observation_id":"fb30fad6-2087-4804-86dd-a88a403aa5df","resolution":{"observed_at":"2026-08-05T10:31:03.024374Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.20937","last_updated":"2025-05-21T11:54:57Z","snapshot_observed_at":"2026-08-07T17:40:12.481199Z","submitted_at":"2025-02-28T10:46:56Z","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","version":2},"cited_work":{"arxiv_id":"2502.20937","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.20937","snapshot_observed_at":"2026-08-05T10:31:02.626069Z","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","venue":"cs.IR","work_id":"8c8c2bce-b9d1-4de7-a718-e99fffdcb68e","year":2025},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.215986Z"},"links":{"cited_paper":"/paper/2502.20937","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:1c950bfae639cb254d83ac532078a381e22d7c5a97f45c5a61d93d232e805fa9","observation_id":"cd6581ea-42a0-4bde-8aa1-16a34cf5048f","resolution":{"observed_at":"2026-08-05T10:31:02.633586Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.005865Z","title":null,"venue":null,"work_id":"a8bdfb1e-f112-4818-951f-c728f9a9489c","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.220869Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:c58c7542e62c9534795796e81a2ea43647e67e13624ada1a0471fcf84070e67f","observation_id":"1928cce6-36f7-44eb-9b8a-6650c08abc75","resolution":{"observed_at":"2026-08-05T10:31:03.010228Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.989717Z","title":"Reuel-Lamparth, A","venue":null,"work_id":"357d1370-36aa-4fbd-bed6-381cd81a2660","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.225854Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:01dc616eb0c0254be246a65b5ba35534c451ef7352342c579992fab76f8ee85e","observation_id":"4a6e8c2e-a0c5-4393-8137-486c69e9eee8","resolution":{"observed_at":"2026-08-05T10:31:02.994446Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.975132Z","title":"Sakaguchi, R","venue":null,"work_id":"3a62f652-271c-4467-adb5-a289fbf9bf80","year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.230285Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:aa75b56cbb65be9cadc20c49a95c051d2e49a9c6e4865eb60c48f3cc7d56f4fb","observation_id":"9de30eba-ef39-4bd6-85ed-2ca5ebc5ff69","resolution":{"observed_at":"2026-08-05T10:31:02.979800Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.234717Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.234717Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:104772be51c557930d67858111281e71c96ef0501b355a366308c9f1137e63cd","observation_id":"8644bea2-f5d8-45ff-8543-8e3c73b0cf41","resolution":{"observed_at":"2026-08-05T10:31:02.234717Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.960501Z","title":"Sanderson","venue":null,"work_id":"0dee9963-5423-47ea-914e-ca0e4b1234ff","year":2010},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.238982Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:3c6cf3e6f79953d1fc87efa28b35fdb650b1066be7063968c123eeada2169cd4","observation_id":"f473b916-536e-47ec-a003-aa68c32eebdf","resolution":{"observed_at":"2026-08-05T10:31:02.964910Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.945740Z","title":"Sclar, Y","venue":null,"work_id":"27637637-9eed-4e8d-be88-d501d2ba2bf5","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.243182Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:45d037cb88a6c36e2d1c628b46e739e2dd66e75bcedabc572655e7cb1363a0a0","observation_id":"c31df391-90e6-4fb8-a8b9-f824cca61ea2","resolution":{"observed_at":"2026-08-05T10:31:02.950379Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.930829Z","title":"Sparck Jones and C","venue":null,"work_id":"0c61af6e-73e5-4f8e-b2fe-b7b8ce6e0232","year":1976},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.248104Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:3e6f76e25af21c54b2629ea5915cc2ba04f1d0d8f8bba618bf674e992139b7a8","observation_id":"b87919ff-4415-45b9-9be5-d6d8977c26cc","resolution":{"observed_at":"2026-08-05T10:31:02.935585Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.253189Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.253189Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:fde2baf40ceca212c6b0f3c62343fb3cd7d00ad414014f546e307205470c3389","observation_id":"b27103e1-f069-4ad6-83b9-9fbcef21a8a4","resolution":{"observed_at":"2026-08-05T10:31:02.253189Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.915009Z","title":null,"venue":null,"work_id":"e137a6e8-817d-4dc5-ad4b-0bc983f1a0c6","year":1998},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.257873Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:c75240e8ac89527bd2bd5d683af2dcbe9d5bc96be99d35b42ea3f1d2f1ef034c","observation_id":"49213a02-9fdd-4668-a7b4-068e583792e8","resolution":{"observed_at":"2026-08-05T10:31:02.920333Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.898800Z","title":null,"venue":null,"work_id":"0dfdc5cb-ebc9-4b6a-9f10-4f0eb1036252","year":2001},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.262534Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:b828d04b8836d6af0a50d5b06efa5878c05dad65f4a54e0b4b54b5133c98d024","observation_id":"3053029f-53cd-4dbf-b20d-ef2e25be86d0","resolution":{"observed_at":"2026-08-05T10:31:02.903240Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.883984Z","title":null,"venue":null,"work_id":"88aad451-0fc0-4507-9822-407de688a67d","year":2001},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.266985Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:0e2cf63bba98203f2e50db2d659cdbc8ddaf9dc57cfdeea6e3b093c6c97a67c4","observation_id":"0ccf09db-f904-40b5-b722-508a892310dc","resolution":{"observed_at":"2026-08-05T10:31:02.888353Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-05T10:31:02.271888Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.271888Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:d22159c0e7e26e353bebbf0473a0fb4601cdbd1306a33b0416fe5a5527c795e3","observation_id":"1d473cba-0ea7-4dcb-88e8-b00755e160ae","resolution":{"observed_at":"2026-08-05T10:31:02.271888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.867837Z","title":null,"venue":null,"work_id":"05630510-2bd8-4770-ad01-71b9b778c190","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.276957Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:d64ca25923ced623694192e227b6bee1bbc1011327feaccca26119814a5caf99","observation_id":"d2b7c9c3-2c39-49ea-bc90-c7c1ebba685c","resolution":{"observed_at":"2026-08-05T10:31:02.873133Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.07682","last_updated":"2022-10-26T05:06:24Z","snapshot_observed_at":"2026-08-02T15:56:35.249569Z","submitted_at":"2022-06-15T17:32:01Z","title":"Emergent Abilities of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.07682","snapshot_observed_at":"2026-08-05T10:31:02.282896Z","title":"Wei et al","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.282896Z"},"links":{"cited_paper":"/paper/2206.07682","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:a16a83b70160f0bf82b27ce1ac9b62df35c7251bade9f9cb48d15aff5c682b05","observation_id":"5a247aea-7324-4277-b3d6-86a6d3e39e04","resolution":{"observed_at":"2026-08-05T10:31:02.282896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.852513Z","title":"Welbl, N","venue":null,"work_id":"003ee5d0-e6ee-4d6f-bfae-7789a886b504","year":2017},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.287920Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:20421de4f3270f79cf8722a3fc93b87bc1350141f79cea5a765c86db3877a799","observation_id":"826811b5-17ba-4d78-a2d8-1eebc9fcec3b","resolution":{"observed_at":"2026-08-05T10:31:02.856887Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.837564Z","title":null,"venue":null,"work_id":"95ad9ef2-9522-462e-bbf9-2189a79c8389","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.292423Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:bc4219c0ec0e4fd5e9061ccc75967fa616375f78a118b227901080dd2da97455","observation_id":"bd8639d5-3661-49c9-9ae8-aeddd79afaae","resolution":{"observed_at":"2026-08-05T10:31:02.842118Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.821602Z","title":"Zellers, A","venue":null,"work_id":"6532334b-a8cd-4e0b-aca0-ff5ef003ac58","year":2019},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.296921Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:a69d6ec23a4ed78789826622280bf5a2a6e5e66df93971affe7c306508f0f4eb","observation_id":"e282fa3b-bb90-4bd8-b2d2-6c7cae47a607","resolution":{"observed_at":"2026-08-05T10:31:02.826115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.805697Z","title":null,"venue":null,"work_id":"101dfbfc-013a-4a83-8573-c4b49efed8e1","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.301739Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:19df4afb414e671aa15effa355f9849f31f3bc1249b3457770595e826b467cb7","observation_id":"bc8a2d74-d951-4f55-ac91-916e6b2b04d7","resolution":{"observed_at":"2026-08-05T10:31:02.811178Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.790712Z","title":"Zheng et al","venue":null,"work_id":"5ba21bb9-a179-4553-babc-2d96a413bf5f","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.306286Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:0d962417795462a2bbcdda99029fb45538971fdc9969b88b31a86ec454343efc","observation_id":"4c487dba-c445-4e33-8390-0e3b677644ce","resolution":{"observed_at":"2026-08-05T10:31:02.795648Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2104.05938","last_updated":"2021-04-13T05:00:35Z","snapshot_observed_at":"2026-08-08T14:49:34.085953Z","submitted_at":"2021-04-13T05:00:35Z","title":"QMSum: A New Benchmark for Query-based Multi-domain Meeting Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.05938","snapshot_observed_at":"2026-08-05T10:31:02.310499Z","title":"Zhong et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.310499Z"},"links":{"cited_paper":"/paper/2104.05938","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:f1049b5931cb4595e8fb26e169f5532cf52c35672e59de2516639ac204faa1e1","observation_id":"01dc6a9e-3429-4a80-ae43-94fbd1f422be","resolution":{"observed_at":"2026-08-05T10:31:02.310499Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17631","last_updated":"2025-03-01T17:06:43Z","snapshot_observed_at":"2026-08-06T12:42:08.815092Z","submitted_at":"2023-10-26T17:48:58Z","title":"JudgeLM: Fine-tuned Large Language Models are Scalable Judges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17631","snapshot_observed_at":"2026-08-05T10:31:02.314947Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.314947Z"},"links":{"cited_paper":"/paper/2310.17631","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:5c757aa93aef22c1e9115c426d18a0b5976656d6dfa0c94816335664d2242976","observation_id":"86795f54-c045-4a17-81a9-74e30fcc2ffc","resolution":{"observed_at":"2026-08-05T10:31:02.314947Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-07-31T19:00:59.311189Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-05T10:31:02.319589Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.319589Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:81e6ed03370a700ec4c7478e38f065a21fc4f661788aa867ab21e32d22f4380a","observation_id":"61763fde-4f72-4bea-a209-5e3cf9f54dc0","resolution":{"observed_at":"2026-08-05T10:31:02.319589Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-08T18:17:02.667191Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs"},"reference_resolution":{"displayed":51,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":27,"verified_exact":1,"verified_fuzzy":23},"total_outbound_references":51},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 51 of 51 outbound references and 8 inbound Pith citation observations for arXiv:2509.04013."}