{"as_of":"2026-08-23T04:11:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:497cc6d6dbe47d7a313b202372056ea8744b0d36be43bcb8250d0a3f145e5501","coverage":[{"denominator":28,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":28,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T00:31:37.128884Z","state":"measured"},{"denominator":38,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":38,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-22T06:32:14.747728+00:00","state":"measured"},{"denominator":10,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":10,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T20:57:42.489401Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"pith","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":2,"observed_at":"2026-08-05T02:28:24.338817Z","source":"pith"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-06T20:57:42.489401Z","title":"An empirical study of llm-as-a-judge: How design choices impact evaluation reliability, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.01413","last_updated":"2025-07-02T07:06:49Z","snapshot_observed_at":"2026-08-16T16:33:11.352892Z","submitted_at":"2025-07-02T07:06:49Z","title":"Evaluating LLM Agent Collusion in Double Auctions","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-06T20:57:42.489401Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2507.01413"},"observation_digest":"sha256:bb4c47ed3adea74af183ed1edc93ec386aa1dfcca4e870fb321ba062f9efb202","observation_id":"4abf00ce-8520-4104-89e0-460594eeca8c","resolution":{"observed_at":"2026-08-06T20:57:42.489401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-04T14:42:50.983413Z","title":"An empirical study of llm-as-a-judge: How design choices impact evaluation reliability.arXiv preprint arXiv:2506.13639,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.23573","last_updated":"2026-05-28T01:14:19Z","snapshot_observed_at":"2026-08-16T17:27:30.568317Z","submitted_at":"2025-09-28T02:08:27Z","title":"Uncovering Vulnerabilities of LLM-Assisted Cyber Threat Intelligence","version":5},"reference_index":2012,"source":"pdf_text","source_observed_at":"2026-08-04T14:42:50.983413Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2509.23573"},"observation_digest":"sha256:71005f2644f7a4fe00bbb368c49b7ff5c87e97328a68b3cc5f7b3fb2a0d64946","observation_id":"5181625e-f7a5-4a69-9903-c0fb3420f29b","resolution":{"observed_at":"2026-08-04T14:42:50.983413Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2605.09121","last_updated":"2026-05-09T19:14:33Z","snapshot_observed_at":"2026-08-08T19:11:23.018391Z","submitted_at":"2026-05-09T19:14:33Z","title":"A Communication-Theoretic Framework for LLM Agents: Cost-Aware Adaptive Reliability","version":1},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-05-12T03:03:05.715652Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2605.09121"},"observation_digest":"sha256:c89657ebb399aec18b1f828a5a3310f458f2c1c0be6861adb9170b3ce0fe7011","observation_id":"94c8a979-7c5b-4517-b045-2a493b8cc496","resolution":{"observed_at":"2026-05-12T03:06:19.529541Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2605.15455","last_updated":"2026-05-14T22:37:56Z","snapshot_observed_at":"2026-08-16T11:10:46.347364Z","submitted_at":"2026-05-14T22:37:56Z","title":"Multi-Turn Neural Transparency: Surfacing Neural Activations Improves User Calibration to LLM Behavioral Drift","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-05-19T14:37:45.304949Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2605.15455"},"observation_digest":"sha256:4bd49213c621895cad654f90745292e151ccff78f4b2974c29fa522dd64b8793","observation_id":"9afa4fa9-d2fd-44cf-87c1-d72498563e7b","resolution":{"observed_at":"2026-05-19T14:42:37.636491Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2605.24319","last_updated":"2026-05-23T00:55:36Z","snapshot_observed_at":"2026-08-14T13:59:28.852731Z","submitted_at":"2026-05-23T00:55:36Z","title":"Omissive Bias in Religious Representation: Benchmarking LLM Answers to Everyday Ethical Decision-making","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-06-30T14:04:51.856877Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2605.24319"},"observation_digest":"sha256:6a33c69ad2a3dd8471df2c7a242780fadfb5717d72f0ed4f1d9771770036a1b8","observation_id":"52999629-05d3-480f-a99f-0b9e298fcd31","resolution":{"observed_at":"2026-06-30T14:14:45.956409Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-07-13T18:51:10.298187Z","title":"Yamauchi, T","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2606.00003","last_updated":"2026-03-25T15:36:15Z","snapshot_observed_at":"2026-08-18T10:00:20.939316Z","submitted_at":"2026-03-25T15:36:15Z","title":"Learning from Mistakes: Can LLM Self-Recover after Misalignment?","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-07-13T18:51:10.298187Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2606.00003"},"observation_digest":"sha256:71ba555e77ddfa83625ffab62a1e34698c8077fd243fb5b2e755b47616949b99","observation_id":"6db26b5f-812e-42eb-a53b-e31c082d7b5c","resolution":{"observed_at":"2026-07-13T18:51:10.298187Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2606.08867","last_updated":"2026-06-07T22:44:00Z","snapshot_observed_at":"2026-08-03T19:07:50.944087Z","submitted_at":"2026-06-07T22:44:00Z","title":"Building Customer Support AI Agents at 100M-User Scale: An Evaluation-Driven Framework","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-06-27T18:21:57.096578Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2606.08867"},"observation_digest":"sha256:95949308d3cc7530792f8c7c3ab6e0df00f111bfc7e74a67b7070f3bd5f264e3","observation_id":"f82759b6-9b63-4e90-9e73-a73c515e77f3","resolution":{"observed_at":"2026-07-02T23:17:29.477423Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2606.09118","last_updated":"2026-07-04T02:42:19Z","snapshot_observed_at":"2026-08-15T16:38:25.826951Z","submitted_at":"2026-06-08T07:11:56Z","title":"ComplexConstraints and Beyond: Expert Rubrics for RLVR","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-06-27T16:37:11.141846Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2606.09118"},"observation_digest":"sha256:4b3b277fe43db50749875b97eb2697352c3a98dc0a385877e1d9205cd9370718","observation_id":"1b36da1e-73a1-4f21-8e99-62204d41cadd","resolution":{"observed_at":"2026-07-03T01:17:31.387512Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2606.11635","last_updated":"2026-06-10T03:56:07Z","snapshot_observed_at":"2026-08-16T17:54:38.322927Z","submitted_at":"2026-06-10T03:56:07Z","title":"Are LLMs Bad at Moral Reasoning?","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-06-27T08:20:24.251540Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2606.11635"},"observation_digest":"sha256:19bdc5ec7b178cdb09fc1b802f5ecacd25704de1a0d584b6cec63ad2dbe6466c","observation_id":"e7d8b4d4-a882-4740-bdaa-f2bdb818f1cb","resolution":{"observed_at":"2026-07-03T13:18:12.618678Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"cited_work":{"arxiv_id":"2506.13639","doi":"10.48550/arxiv.2506.13639","metadata_source":"pith","pith_arxiv_id":"2506.13639","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2506.13639 , year =","venue":"cs.CL","work_id":"72b5917c-1851-4031-a0bb-8f0bff215fa5","year":2025},"citing_paper":{"arxiv_id":"2607.08535","last_updated":"2026-07-09T14:31:05Z","snapshot_observed_at":"2026-08-18T20:55:46.907256Z","submitted_at":"2026-07-09T14:31:05Z","title":"When the Judge Changes, So Does the Measurement: Auditing LLM-as-Judge Reliability","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-07-10T05:47:32.670216Z"},"links":{"cited_paper":"/paper/2506.13639","citing_paper":"/paper/2607.08535"},"observation_digest":"sha256:1e14b32513897cf736dde1098f02340cc1d6bfc34a21babaa230eb51e4882d82","observation_id":"2037b41f-1d8a-41fa-9a14-187003592721","resolution":{"observed_at":"2026-07-10T05:56:50.477026Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-18T01:21:02.044319+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2506.13639/citation-record","integrity":"/paper/2506.13639/integrity","json":"/paper/2506.13639/citation-record.json","paper":"/paper/2506.13639"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T00:31:38.173345Z","title":"InProceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, EMNLP 2024, Miami, FL, USA, November 12-16, 2024, pages 8301–8327","venue":null,"work_id":"89b5bfb4-eab3-4cca-acde-63fd8dfeb79d","year":2024},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:34.852140Z"},"links":{"citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:1e499004ebca964c0a29f1d4645a50cea99df53331d10532c50c503657347f36","observation_id":"4ac6696d-1805-40bd-9dcd-954d938735fd","resolution":{"observed_at":"2026-08-07T00:31:38.221562Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.19437","last_updated":"2025-02-18T17:26:38Z","snapshot_observed_at":"2026-08-18T18:18:37.449517Z","submitted_at":"2024-12-27T04:03:16Z","title":"DeepSeek-V3 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.19437","snapshot_observed_at":"2026-08-07T00:31:34.925180Z","title":"Sumanth Doddapaneni, Mohammed Safi Ur Rahman Khan, Sshubam Verma, and Mitesh M","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:34.925180Z"},"links":{"cited_paper":"/paper/2412.19437","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:700155bef540c5efefe1e698bfd77dd6ac73e3388c3c3e78f1c506591dacc2c4","observation_id":"3342a1ff-0da4-4c63-886e-f002b45b3315","resolution":{"observed_at":"2026-08-07T00:31:34.925180Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.13439","last_updated":"2024-11-26T06:18:16Z","snapshot_observed_at":"2026-08-16T13:41:30.252719Z","submitted_at":"2024-06-19T10:59:48Z","title":"Finding Blind Spots in Evaluator LLMs with Interpretable Checklists","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.13439","snapshot_observed_at":"2026-08-07T00:31:35.022146Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.022146Z"},"links":{"cited_paper":"/paper/2406.13439","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:5b84ac5cc17fc3c62f424cabe9ca0d6f21941cc5df9c76044b321babff4e34f8","observation_id":"f636348e-0aa3-4358-9ac0-529ed93c7a90","resolution":{"observed_at":"2026-08-07T00:31:35.022146Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-07T00:31:35.146627Z","title":"Yann Dubois, Balázs Galambosi, Percy Liang, and Tat- sunori B","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.146627Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:e86dc29fd3cf92c03916a21e39157f0e2bd4170e5873a4631ff8489c2024261e","observation_id":"8e604453-680f-4eaa-a9fb-b77ef8f92ea3","resolution":{"observed_at":"2026-08-07T00:31:35.146627Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04475","last_updated":"2025-03-10T09:27:03Z","snapshot_observed_at":"2026-07-06T17:56:23.317089Z","submitted_at":"2024-04-06T02:29:02Z","title":"Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.04475","snapshot_observed_at":"2026-08-07T00:31:35.214458Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.214458Z"},"links":{"cited_paper":"/paper/2404.04475","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:0ab122f39b236e06136bbcc51f142a89c6ec89ca367e4e58d9edeaaa3b545d59","observation_id":"e29f9c5b-0188-49b5-bae4-ae976161266a","resolution":{"observed_at":"2026-08-07T00:31:35.214458Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15594","last_updated":"2025-10-19T10:32:43Z","snapshot_observed_at":"2026-08-20T12:52:29.141935Z","submitted_at":"2024-11-23T16:03:35Z","title":"A Survey on LLM-as-a-Judge","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15594","snapshot_observed_at":"2026-08-07T00:31:35.273814Z","title":"Jiawei Gu, Xuhui Jiang, Zhichao Shi, Hexiang Tan, Xuehao Zhai, Chengjin Xu, Wei Li, Yinghan Shen, Shengjie Ma, Honghao Liu, Yuanzhuo Wang, and Jian Guo","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.273814Z"},"links":{"cited_paper":"/paper/2411.15594","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:872938f49ae5d5186785d726ec528a6d8e7375a3c410a9ccb7f3cbbd7662f17e","observation_id":"dbc8685d-4bd6-4dff-b760-dfcb40673684","resolution":{"observed_at":"2026-08-07T00:31:35.273814Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04088","last_updated":"2024-01-08T18:47:34Z","snapshot_observed_at":"2026-08-13T19:43:49.936776Z","submitted_at":"2024-01-08T18:47:34Z","title":"Mixtral of Experts","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04088","snapshot_observed_at":"2026-08-07T00:31:35.408153Z","title":"Ryo Kamoi, Yusen Zhang, Nan Zhang, Jiawei Han, and Rui Zhang","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.408153Z"},"links":{"cited_paper":"/paper/2401.04088","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:fc7dac37a76058b4b2ad140ce4bc08a8c3e4e26518f15cd0afc024191d017d99","observation_id":"3d2d45e3-c5f7-4aaa-af0a-d45d682d4127","resolution":{"observed_at":"2026-08-07T00:31:35.408153Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01297","last_updated":"2024-12-03T19:14:06Z","snapshot_observed_at":"2026-08-18T10:44:14.134406Z","submitted_at":"2024-06-03T13:05:46Z","title":"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01297","snapshot_observed_at":"2026-08-07T00:31:35.477675Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.477675Z"},"links":{"cited_paper":"/paper/2406.01297","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:84ae714d587fbb05097e47d9cc5a1bd6765715bf8aebdda03fd2d3f336d0bbb4","observation_id":"4ccd1d5c-745c-4c72-a8c4-a1c8588a3896","resolution":{"observed_at":"2026-08-07T00:31:35.477675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.05761","last_updated":"2025-03-25T07:12:10Z","snapshot_observed_at":"2026-08-19T05:43:17.336704Z","submitted_at":"2024-06-09T12:30:30Z","title":"The BiGGen Bench: A Principled Benchmark for Fine-grained Evaluation of Language Models with Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.05761","snapshot_observed_at":"2026-08-07T00:31:35.564117Z","title":"Klaus Krippendorff","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.564117Z"},"links":{"cited_paper":"/paper/2406.05761","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:ed241358ceb62c809f7849839b835bac0cbbd83cfe18dbb149e2388de2ffa67d","observation_id":"840b7fc9-1e0e-4dcc-99f5-49bfd2af4791","resolution":{"observed_at":"2026-08-07T00:31:35.564117Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T00:31:35.904893Z","title":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei- Jing Zhu","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.904893Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:36dc02cdbe6082976f1cc85bcf02dbd5e16b179532c234d964084585e79d8c12","observation_id":"c6e41943-29f7-41ca-b7c5-9d96d54aeec3","resolution":{"observed_at":"2026-08-07T00:31:35.904893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-20T17:50:08.381556Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-07T00:31:36.166184Z","title":"Yuda Song, Hanlin Zhang, Carson Eisenach, Sham M","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.166184Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:3679508bb98b760435bf73c9fb9e1e8b5a296cb0a17c1fefabac685b706cccc6","observation_id":"377d08fb-1e9b-42ab-91c7-0b051cccca8d","resolution":{"observed_at":"2026-08-07T00:31:36.166184Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.02674","last_updated":"2025-02-25T16:59:11Z","snapshot_observed_at":"2026-08-14T20:58:55.959220Z","submitted_at":"2024-12-03T18:47:26Z","title":"Mind the Gap: Examining the Self-Improvement Capabilities of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.02674","snapshot_observed_at":"2026-08-07T00:31:36.251369Z","title":"Qwen Team","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.251369Z"},"links":{"cited_paper":"/paper/2412.02674","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:19016eafa928306cd91cbff33ab188196363d8aae78b9e3f551358cc6052024f","observation_id":"e2a5c216-4747-45e8-91ec-72ca80aed3cd","resolution":{"observed_at":"2026-08-07T00:31:36.251369Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T00:31:37.813176Z","title":"InThe Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5,","venue":null,"work_id":"a6721006-db33-422f-814e-fc88770b05b2","year":2023},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.563627Z"},"links":{"citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:5bd72c0c32781ce3e913a4e915272215fef301419a49bea0948d89509f068a8f","observation_id":"8e8ff90b-378a-49fc-a3c1-5e875cc9bbc9","resolution":{"observed_at":"2026-08-07T00:31:37.916376Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-08-18T13:17:17.701561Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-07T00:31:36.601561Z","title":"Ziyou Yan","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.601561Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:27947c842a9a28456bc71bf28abca8784735102e2ccbae7b15be669040b88fd5","observation_id":"60835722-30b9-4029-95ec-185286065f5e","resolution":{"observed_at":"2026-08-07T00:31:36.601561Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-08-17T18:50:07.059564Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-07T00:31:36.692401Z","title":"Jiayi Ye, Yanbo Wang, Yue Huang, Dongping Chen, Qihui Zhang, Nuno Moniz, Tian Gao, Werner Geyer, Chao Huang, Pin-Yu Chen, Nitesh V","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.692401Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:6ba7ea0dad3f32013de445057dafdf19541f7366ebf0f3390e306996dce13d85","observation_id":"631e4c20-e8c3-48bd-9989-3bcb3f7350d9","resolution":{"observed_at":"2026-08-07T00:31:36.692401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02736","last_updated":"2024-10-04T03:57:47Z","snapshot_observed_at":"2026-08-21T19:39:41.738597Z","submitted_at":"2024-10-03T17:53:30Z","title":"Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02736","snapshot_observed_at":"2026-08-07T00:31:36.795367Z","title":"Weizhe Yuan, Richard Yuanzhe Pang, Kyunghyun Cho, Xian Li, Sainbayar Sukhbaatar, Jing Xu, and Ja- son Weston","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.795367Z"},"links":{"cited_paper":"/paper/2410.02736","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:04a66db233d16a3858a12b4c63140f60a20ac639b4e53725563ae09351aca306","observation_id":"231646de-e959-454f-8438-dadbef107fb7","resolution":{"observed_at":"2026-08-07T00:31:36.795367Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T00:31:37.612863Z","title":"InForty-first International Conference on Machine Learning, ICML 2024, Vienna, Austria, July 21-27,","venue":null,"work_id":"f4c7b72e-fe87-41d7-85c1-cadd6679707a","year":2024},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.882454Z"},"links":{"citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:1ab92a20b7933e6986d325771367f36b52583d6585a951a640ed6b9bfe89d2b1","observation_id":"378b0edc-7941-49ea-9496-d35e4823bf24","resolution":{"observed_at":"2026-08-07T00:31:37.729676Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1904.09675","last_updated":"2020-02-24T18:59:28Z","snapshot_observed_at":"2026-07-29T15:42:51.774083Z","submitted_at":"2019-04-21T23:08:53Z","title":"BERTScore: Evaluating Text Generation with BERT","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.09675","snapshot_observed_at":"2026-08-07T00:31:36.976720Z","title":"Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric P","venue":null,"work_id":null,"year":1904},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.976720Z"},"links":{"cited_paper":"/paper/1904.09675","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:ac86e0e81c6622ec7db61856c8d9e7033af5a0754d7d852a7d1b3ea00b0808ff","observation_id":"95a9b814-a5ff-43fc-8d30-8d1b280ba898","resolution":{"observed_at":"2026-08-07T00:31:36.976720Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-07T00:31:37.043951Z","title":"Preprint, arXiv:2306.05685","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:37.043951Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:330ca0944ee212dc60019dfedf2b6d80517f65cb3d67346ba9ad2d303da687df","observation_id":"d6713a3b-c396-46e6-8795-96fe2f5bc088","resolution":{"observed_at":"2026-08-07T00:31:37.043951Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T00:31:37.508776Z","title":null,"venue":null,"work_id":"060c0ce8-fd65-4cc2-b0de-5dfb4dc6c258","year":2024},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:37.128884Z"},"links":{"citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:58ee06911a1b90a4697348486041dce065b716df0a22ba35e44253128c8cd8b2","observation_id":"9a2a6f2e-a85c-4faa-95fe-8e40091b9e68","resolution":{"observed_at":"2026-08-07T00:31:37.558801Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.06551","last_updated":"2024-10-07T08:49:49Z","snapshot_observed_at":"2026-08-20T11:42:02.365712Z","submitted_at":"2024-07-09T05:16:22Z","title":"OffsetBias: Leveraging Debiased Data for Tuning Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.06551","snapshot_observed_at":"2026-08-07T00:31:36.029108Z","title":"InProceedings of the 40th Annual Meeting of the Association for Compu- tational Linguistics, pages 311–318, Philadelphia, Pennsylvania, USA","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2002,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.029108Z"},"links":{"cited_paper":"/paper/2407.06551","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:28a2c0acc23d6bf00e23507a44e281201f9bfa0df3ee94ff07b6169cdd164195","observation_id":"c125dec4-740c-4de0-b143-a7f6f052dd4f","resolution":{"observed_at":"2026-08-07T00:31:36.029108Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11830","last_updated":"2023-12-08T02:57:26Z","snapshot_observed_at":"2026-08-16T14:58:53.540263Z","submitted_at":"2023-09-21T07:07:49Z","title":"Goal-Oriented Prompt Attack and Safety Evaluation for LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11830","snapshot_observed_at":"2026-08-07T00:31:35.732858Z","title":"InText Summariza- tion Branches Out, pages 74–81, Barcelona, Spain","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2004,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.732858Z"},"links":{"cited_paper":"/paper/2309.11830","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:de019a87ca8ee28b5e140e84c257a33c2aa034c02accff93d9ff7ccbb85d4f84","observation_id":"0c62f108-f991-4eef-a9d4-91eaa0926d9f","resolution":{"observed_at":"2026-08-07T00:31:35.732858Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05579","last_updated":"2024-12-10T05:49:12Z","snapshot_observed_at":"2026-08-17T10:18:05.650518Z","submitted_at":"2024-12-07T08:07:24Z","title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05579","snapshot_observed_at":"2026-08-07T00:31:35.647892Z","title":"Haitao Li, Qian Dong, Junjie Chen, Huixue Su, Yujia Zhou, Qingyao Ai, Ziyi Ye, and Yiqun Liu","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2011,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.647892Z"},"links":{"cited_paper":"/paper/2412.05579","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:c9202dfbb519bbf52cd5079cafcd139f6359ef7b563ae0a8746e7bd7dfe550db","observation_id":"65da1aa6-5c74-41db-969d-19ad9127f48d","resolution":{"observed_at":"2026-08-07T00:31:35.647892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1411.5726","last_updated":"2015-06-03T01:42:20Z","snapshot_observed_at":"2026-08-14T23:10:48.763001Z","submitted_at":"2014-11-20T23:54:35Z","title":"CIDEr: Consensus-based Image Description Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1411.5726","snapshot_observed_at":"2026-08-07T00:31:36.490899Z","title":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc V","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2015,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.490899Z"},"links":{"cited_paper":"/paper/1411.5726","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:f1a54216a3fcd15929d6cd4f23ec318a464e3ab84dfea612251ae6744a34b265","observation_id":"92140abe-0aa6-4d6b-88e3-032f50a74885","resolution":{"observed_at":"2026-08-07T00:31:36.490899Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.09025","last_updated":"2020-10-19T14:10:10Z","snapshot_observed_at":"2026-08-18T05:05:12.754805Z","submitted_at":"2020-09-18T18:54:15Z","title":"COMET: A Neural Framework for MT Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.09025","snapshot_observed_at":"2026-08-07T00:31:36.112514Z","title":"Kayla Schroeder and Zach Wood-Doughty","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.112514Z"},"links":{"cited_paper":"/paper/2009.09025","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:ef1407f4da2fb397ea86118447c0f4c4398a468f46a526475595244b0003a6a7","observation_id":"f4b84ba2-f15a-4f92-8c20-7ce6e8b4046a","resolution":{"observed_at":"2026-08-07T00:31:36.112514Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T00:31:38.013360Z","title":"InAdvances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Sys- tems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16,","venue":null,"work_id":"48b2ee4f-67ae-44a1-924b-d58797bedcd0","year":2023},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:35.832802Z"},"links":{"citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:2a60fdcd02516ef3dc083697bc4d0adf34c39f3b3b4a04c16036c4fba167b142","observation_id":"6a031a9b-11da-440f-91ab-11ddad9dbc81","resolution":{"observed_at":"2026-08-07T00:31:38.083315Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.14219","last_updated":"2024-08-30T21:17:17Z","snapshot_observed_at":"2026-08-17T03:25:04.404839Z","submitted_at":"2024-04-22T14:32:33Z","title":"Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.14219","snapshot_observed_at":"2026-08-07T00:31:34.708896Z","title":"CoRR, abs/2404.14219","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:34.708896Z"},"links":{"cited_paper":"/paper/2404.14219","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:99bc95bcb7d8256fab4d55fb5f20faa6e6409a5c3c04f77501361126635cb113","observation_id":"e54480e0-3d9a-48ea-bd97-8f6f87743c9c","resolution":{"observed_at":"2026-08-07T00:31:34.708896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12624","last_updated":"2025-08-18T00:07:23Z","snapshot_observed_at":"2026-08-19T05:45:27.077361Z","submitted_at":"2024-06-18T13:49:54Z","title":"Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12624","snapshot_observed_at":"2026-08-07T00:31:36.353198Z","title":"Ramakrishna Vedantam, C","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.353198Z"},"links":{"cited_paper":"/paper/2406.12624","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:f2b721ee2c6173605a270233fd9e27c5e59c26c489665f659e808d7fdbf56b33","observation_id":"dbccbcef-a0e0-4a25-a3d0-e0ab3cfabc98","resolution":{"observed_at":"2026-08-07T00:31:36.353198Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-16T19:16:57.442306Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability"},"reference_resolution":{"displayed":28,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":24,"verified_exact":0,"verified_fuzzy":4},"total_outbound_references":28},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"thesis":"As of 23 August 2026, this Paper Citation Record lists 28 of 28 outbound references and 10 inbound Pith citation observations for arXiv:2506.13639."}