{"as_of":"2026-08-15T19:54:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:2e8f4d6392e60c6ab9f6a6a4c962dc0bca53983ab682f1f9f6a6a6a89c2ce031","coverage":[{"denominator":76,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":76,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-11T14:04:26.439792Z","state":"measured"},{"denominator":104,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":104,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":28,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":28,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T14:01:13.015050Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"arxiv_reference","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":5,"observed_at":"2026-08-05T02:28:24.338817Z","source":"arxiv_reference"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-07T14:01:13.015050Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge.arXiv preprint arXiv:2412.12509, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.20296","last_updated":"2025-05-26T17:59:53Z","snapshot_observed_at":"2026-08-14T23:04:03.827146Z","submitted_at":"2025-05-26T17:59:53Z","title":"Reasoning LLMs are Wandering Solution Explorers","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T14:01:13.015050Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2505.20296"},"observation_digest":"sha256:cf666c72f0a21f4ae49ee2b38e48b88b91991b792890d0b5ff74f0103459fb0b","observation_id":"7aa2c22f-8847-4e96-a123-cca44ace6ce4","resolution":{"observed_at":"2026-08-07T14:01:13.015050Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-07T00:31:36.166184Z","title":"Yuda Song, Hanlin Zhang, Carson Eisenach, Sham M","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.13639","last_updated":"2025-06-16T16:04:43Z","snapshot_observed_at":"2026-08-07T00:26:19.217552Z","submitted_at":"2025-06-16T16:04:43Z","title":"An Empirical Study of LLM-as-a-Judge: How Design Choices Impact Evaluation Reliability","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T00:31:36.166184Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2506.13639"},"observation_digest":"sha256:4533db7b51733f92cfaeb9391f8b4b6de088386bf3f12a62f431cffb050fba15","observation_id":"377d08fb-1e9b-42ab-91c7-0b051cccca8d","resolution":{"observed_at":"2026-08-07T00:31:36.166184Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-06T11:16:04.910071Z","title":"Schroeder and Z","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.22879","last_updated":"2025-07-31T16:54:43Z","snapshot_observed_at":"2026-08-14T03:40:36.016201Z","submitted_at":"2025-07-30T17:55:06Z","title":"RecGPT Technical Report","version":2},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-06T11:16:04.910071Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2507.22879"},"observation_digest":"sha256:44347bd4fd5a53064fc230801b2f0a58ca215584651698b6a22ffc7fc80b2e1a","observation_id":"89508a75-c5c4-4b99-a2e0-b798d2b9cdeb","resolution":{"observed_at":"2026-08-06T11:16:04.910071Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T23:09:11.546963Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.05938","last_updated":"2025-08-08T02:04:14Z","snapshot_observed_at":"2026-08-13T18:38:43.647004Z","submitted_at":"2025-08-08T02:04:14Z","title":"Prosocial Behavior Detection in Player Game Chat: From Aligning Human-AI Definitions to Efficient Annotation at Scale","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-05T23:09:11.546963Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2508.05938"},"observation_digest":"sha256:769539f19e1b606d1913e19348544f6664a85a273cf4217e749acd43e7686b8e","observation_id":"be00877a-4412-41e5-8544-11daeb8df5d2","resolution":{"observed_at":"2026-08-05T23:09:11.546963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2508.15503","last_updated":"2026-05-21T21:46:55Z","snapshot_observed_at":"2026-08-14T19:16:27.854742Z","submitted_at":"2025-08-21T12:30:30Z","title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","version":5},"reference_index":118,"source":"pdf_text","source_observed_at":"2026-05-18T22:02:36.307598Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2508.15503"},"observation_digest":"sha256:5d6efc850dc18c25811d56192a000f0b3b0929dff11a5220c1424c880f6cdc0e","observation_id":"9f0165a7-90cd-4660-b6ab-46638cbf535c","resolution":{"observed_at":"2026-05-18T22:02:52.228552Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2508.15503","last_updated":"2026-05-21T21:46:55Z","snapshot_observed_at":"2026-08-14T19:16:27.854742Z","submitted_at":"2025-08-21T12:30:30Z","title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","version":6},"reference_index":118,"source":"pdf_text","source_observed_at":"2026-05-25T08:18:18.448122Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2508.15503"},"observation_digest":"sha256:a12cbd933f379e19a71e7ec8bc6c7dd1a1a34197462c7698becec8521f83257d","observation_id":"1a1fe4f8-e899-43c2-91dd-807fc59c67b1","resolution":{"observed_at":"2026-05-25T08:20:31.427886Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T11:26:27.495885Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.02855","last_updated":"2025-09-02T21:58:58Z","snapshot_observed_at":"2026-08-12T12:27:28.613494Z","submitted_at":"2025-09-02T21:58:58Z","title":"IDEAlign: Comparing Large Language Models to Human Experts in Open-ended Interpretive Annotations","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-05T11:26:27.495885Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2509.02855"},"observation_digest":"sha256:22e1eafdcbdc84720d9352eab05b6d7d367e8ad790c73de091737d0a38abce2c","observation_id":"5f0f1c80-3e02-483f-889d-98728815c6f1","resolution":{"observed_at":"2026-08-05T11:26:27.495885Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-04T14:42:30.591819Z","title":"Can you trust llm judgments? reliability of llm-as-a-judge","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2510.05115","last_updated":"2026-05-29T07:07:01Z","snapshot_observed_at":"2026-08-14T05:21:41.109881Z","submitted_at":"2025-09-28T12:25:31Z","title":"SAC-Opt: Semantic Anchors for Iterative Correction in Optimization Modeling","version":3},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-04T14:42:30.591819Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2510.05115"},"observation_digest":"sha256:c61af3d313385267389fdc28f9f4c34408cd0ac694f4451875e84624d1570c1d","observation_id":"210cc5ad-3c76-4b9c-acb3-9ff08272e63f","resolution":{"observed_at":"2026-08-04T14:42:30.591819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2512.10687","last_updated":"2026-04-20T11:46:57Z","snapshot_observed_at":"2026-08-11T08:27:08.305717Z","submitted_at":"2025-12-11T14:34:40Z","title":"Safe for Whom? Rethinking How We Evaluate the Safety of LLMs for Real Users","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-05-16T23:22:42.431997Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2512.10687"},"observation_digest":"sha256:9c1b026246fde9e8f7e244b6c58cc318b7472d82ceebd072324680947a321a38","observation_id":"cf681782-f8ee-4fa3-84e2-41dc5df3ab22","resolution":{"observed_at":"2026-05-16T23:23:39.892183Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2604.05371","last_updated":"2026-04-07T03:16:44Z","snapshot_observed_at":"2026-07-06T22:54:08.897942Z","submitted_at":"2026-04-07T03:16:44Z","title":"LLM-as-Judge for Semantic Judging of Powerline Segmentation in UAV Inspection","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-10T19:37:19.360378Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.05371"},"observation_digest":"sha256:da1a0474f647f46ddc434db02c486ae2519a4783f79673a604e9e5259a4a212b","observation_id":"9a234b33-9b17-4179-b58a-432ce1af1df6","resolution":{"observed_at":"2026-05-10T22:45:48.058694Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2604.05593","last_updated":"2026-04-07T08:43:30Z","snapshot_observed_at":"2026-08-12T23:25:06.692122Z","submitted_at":"2026-04-07T08:43:30Z","title":"Label Effects: Shared Heuristic Reliance in Trust Assessment by Humans and LLM-as-a-Judge","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-05-10T19:38:11.595077Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.05593"},"observation_digest":"sha256:81baf8a4f0bcad974ac1f0e325966759e3e4f832327842d32cd2616efdcb5666","observation_id":"b2143566-bc8f-46b0-babc-aadb24da9869","resolution":{"observed_at":"2026-05-10T22:40:51.745869Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2604.15326","last_updated":"2026-03-06T02:38:18Z","snapshot_observed_at":"2026-08-13T20:20:16.427686Z","submitted_at":"2026-03-06T02:38:18Z","title":"Analyzing the Presentation, Content, and Utilization of References in LLM-powered Conversational AI Systems","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-05-15T15:59:41.632090Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.15326"},"observation_digest":"sha256:65ab6748cba6ad7226a445551b70eca31b2b0be91eacf2d42bfd8ae8c69fdc18","observation_id":"9e4c2215-202a-4f8d-9dc3-e3495f1de088","resolution":{"observed_at":"2026-05-15T16:00:10.111008Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2604.16706","last_updated":"2026-08-10T01:18:04Z","snapshot_observed_at":"2026-08-15T03:40:39.253330Z","submitted_at":"2026-04-17T21:15:35Z","title":"Auditing Automated Evaluation, Error Propagation, and Runtime Mitigation in Tool-Using Language Agents","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-05-10T08:00:45.789649Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.16706"},"observation_digest":"sha256:13cb6884c6aac7ef548d816638983ffb46266526a8cf39eee6b52045d70ccae0","observation_id":"a506623b-98f4-4159-9a21-1e3775c73ae6","resolution":{"observed_at":"2026-05-10T08:02:24.999729Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-07-12T19:18:49.672005Z","title":"arXiv:2412.12509","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2604.16707","last_updated":"2026-06-21T14:58:53Z","snapshot_observed_at":"2026-08-14T15:22:14.408821Z","submitted_at":"2026-04-17T21:18:06Z","title":"Mixed response geometry and critical crossover in the Ising model","version":3},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-07-12T19:18:49.672005Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.16707"},"observation_digest":"sha256:285c4a43146af027d6e947d5a54e8aac0f65858b327a44de1da76fd19b5425f6","observation_id":"9bc9f2bf-31e1-444d-9fd4-8a6c4a14de61","resolution":{"observed_at":"2026-07-12T19:18:49.672005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2604.20652","last_updated":"2026-04-23T15:40:50Z","snapshot_observed_at":"2026-08-14T20:25:25.642076Z","submitted_at":"2026-04-22T15:03:37Z","title":"Large Language Models Outperform Humans in Fraud Detection and Resistance to Motivated Investor Pressure","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-05-09T23:40:00.572491Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2604.20652"},"observation_digest":"sha256:3f8bccd94b71376bd5c28fada31a7a4502ce564f8be5490e6b9cc7f88b78567c","observation_id":"fbbfbae8-c536-4762-84c9-9b5bbcfecc41","resolution":{"observed_at":"2026-05-11T14:06:02.544064Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.07032","last_updated":"2026-07-20T17:49:20Z","snapshot_observed_at":"2026-08-02T14:41:04.240876Z","submitted_at":"2026-05-07T23:22:07Z","title":"A Systematic Investigation of RL-Jailbreaking in LLMs","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-11T01:32:42.151644Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.07032"},"observation_digest":"sha256:5793737a19d9dd60da987c949c7f6f2e215e67d26a2891ddc954afeb167468b2","observation_id":"1e9fd0f6-791c-4bac-80d2-3532eb091302","resolution":{"observed_at":"2026-05-11T01:40:52.619473Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.07032","last_updated":"2026-07-20T17:49:20Z","snapshot_observed_at":"2026-08-02T14:41:04.240876Z","submitted_at":"2026-05-07T23:22:07Z","title":"A Systematic Investigation of RL-Jailbreaking in LLMs","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-06-30T22:59:07.861941Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.07032"},"observation_digest":"sha256:64c30cd219fe74e67ec9b3e1f3435f1e5e9fe54d8099db9fda0f2a45d1e8f0ac","observation_id":"f95f96aa-bc93-4936-935e-bf0f03a6df6e","resolution":{"observed_at":"2026-07-01T13:35:46.505086Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-02T14:41:06.754830Z","title":"and Wood-Doughty, Z","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2605.07032","last_updated":"2026-07-20T17:49:20Z","snapshot_observed_at":"2026-08-02T14:41:04.240876Z","submitted_at":"2026-05-07T23:22:07Z","title":"A Systematic Investigation of RL-Jailbreaking in LLMs","version":3},"reference_index":2026,"source":"pdf_text","source_observed_at":"2026-08-02T14:41:06.754830Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.07032"},"observation_digest":"sha256:25805a5dfb48311ed10979aa1ad2b16b24df794f6da2ef71587603016b473882","observation_id":"073fcf87-a936-4e32-8649-d782e3a5e264","resolution":{"observed_at":"2026-08-02T14:41:06.754830Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.09121","last_updated":"2026-05-09T19:14:33Z","snapshot_observed_at":"2026-08-08T19:11:23.018391Z","submitted_at":"2026-05-09T19:14:33Z","title":"A Communication-Theoretic Framework for LLM Agents: Cost-Aware Adaptive Reliability","version":1},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-05-12T03:03:05.715652Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.09121"},"observation_digest":"sha256:275ec52343d4f6253344c14fe8496817d971b69b91ba07610eb0540454ab1c31","observation_id":"5fa0f76a-a8f6-4f10-95b4-4040b442bef9","resolution":{"observed_at":"2026-05-12T03:06:19.501659Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.19035","last_updated":"2026-08-07T08:13:03Z","snapshot_observed_at":"2026-08-15T09:05:15.962456Z","submitted_at":"2026-05-18T18:57:54Z","title":"Trustworthy Agent Network: Trust in Agent Networks Must Be Baked In, Not Bolted On","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-05-20T10:31:16.368065Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.19035"},"observation_digest":"sha256:8b9bead2c00ee12c9e97313ba69e39c3b54d1a906970ecaf8cafad9610b1d76d","observation_id":"c054c141-fd1e-42c2-b791-d2cc8bdb2c31","resolution":{"observed_at":"2026-05-20T10:33:12.402693Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.24279","last_updated":"2026-05-22T23:13:21Z","snapshot_observed_at":"2026-08-14T00:32:02.574383Z","submitted_at":"2026-05-22T23:13:21Z","title":"ContextEcho: A Benchmark for Persona Drift in Long Agentic-Coding Sessions","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-06-30T15:17:37.904831Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.24279"},"observation_digest":"sha256:80da2a94dba6c876e5996715778ecd3484d1e55c3cf031d4f8c91d8fc88441af","observation_id":"2a5a7f47-d2ca-40f3-8387-c5a8392c2013","resolution":{"observed_at":"2026-06-30T15:34:48.458667Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2605.25440","last_updated":"2026-05-25T05:31:44Z","snapshot_observed_at":"2026-08-06T20:23:39.023951Z","submitted_at":"2026-05-25T05:31:44Z","title":"A Multi-Agent LLM Framework for Rating the Quality of Surgical Feedback","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-06-29T22:27:26.911873Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2605.25440"},"observation_digest":"sha256:bc88938ea570a76698362bc71381bee4e44859b3bacf6fdb0195127c2c4767ba","observation_id":"0d6cb879-eb2b-49f8-b934-f36a4d892ec7","resolution":{"observed_at":"2026-06-29T22:34:01.998548Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2606.09118","last_updated":"2026-07-04T02:42:19Z","snapshot_observed_at":"2026-08-15T16:38:25.826951Z","submitted_at":"2026-06-08T07:11:56Z","title":"ComplexConstraints and Beyond: Expert Rubrics for RLVR","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-06-27T16:37:11.141846Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2606.09118"},"observation_digest":"sha256:761f7a0f7d51f6e3f9ae2a52debce260e979d60ba560a4d8b2a87c182b0b7520","observation_id":"df36f15d-1dab-42b4-98e9-0144a15dea3e","resolution":{"observed_at":"2026-07-03T01:17:31.332133Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2606.24428","last_updated":"2026-06-23T11:05:05Z","snapshot_observed_at":"2026-08-14T15:38:39.894094Z","submitted_at":"2026-06-23T11:05:05Z","title":"Escaping the Self-Confirmation Trap: An Execute-Distill-Verify Paradigm for Agentic Experience Learning","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-06-25T23:49:38.932474Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2606.24428"},"observation_digest":"sha256:ba08a03aef5d41c438a7b7b62f0c109d61b0f66e80f23ef9337bd22194bfd263","observation_id":"fc3ef303-5c24-48e1-a078-e054506492d3","resolution":{"observed_at":"2026-07-04T17:19:59.995041Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":"2412.12509","doi":"10.48550/arxiv.2412.12509","metadata_source":"arxiv_reference","pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Can you trust llm judgments? reliability of llm-as- a-judge","venue":"arXiv (Cornell University)","work_id":"fbc020bc-197e-43ce-92fd-3dca29743d27","year":2024},"citing_paper":{"arxiv_id":"2606.30987","last_updated":"2026-06-29T23:51:09Z","snapshot_observed_at":"2026-08-15T08:39:45.972140Z","submitted_at":"2026-06-29T23:51:09Z","title":"Measuring Judgment Quality in Natural-Language Explanations: Evidence from Forecasting Tournaments","version":1},"reference_index":124,"source":"arxiv_source","source_observed_at":"2026-07-01T01:18:53.736430Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2606.30987"},"observation_digest":"sha256:0477b7ebf80296869c4067e78f35f7694894b3d510f27d9df3715ff5daf484d3","observation_id":"58858e04-0d09-4ed6-8eb9-fe3cb0179867","resolution":{"observed_at":"2026-07-01T13:05:44.282114Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-01T08:46:15.119758Z","title":"Can you trust llm judgments? reliability of llm-as-a-judge","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.21010","last_updated":"2026-07-23T07:48:20Z","snapshot_observed_at":"2026-08-09T19:59:33.109243Z","submitted_at":"2026-07-23T07:48:20Z","title":"Reexamining zero-shot summarization: Empirical investigation of trustworthiness of LLM-summarizers","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-01T08:46:15.119758Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2607.21010"},"observation_digest":"sha256:839f26a0894e4199cebd0648c3a6d529560f6600390a70d3cc1428b3b45dc6d6","observation_id":"7de55c08-2825-4751-9ff4-6e4fcd5c413a","resolution":{"observed_at":"2026-08-01T08:46:15.119758Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-02T12:49:10.833195Z","title":"Can you trust llm judgments? reliability of llm-as-a-judge.arXiv preprint arXiv:2412.125092024,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-13T00:19:20.192945Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.833195Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:da12e7049a6833146d829065c7167dceda80f559e29b21c4fbc31b598b34ccda","observation_id":"98c7724a-5157-4c4d-a965-2f801e888550","resolution":{"observed_at":"2026-08-02T12:49:10.833195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-04T21:00:08.279354Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.01783","last_updated":"2026-08-03T06:58:51Z","snapshot_observed_at":"2026-08-13T09:54:32.558734Z","submitted_at":"2026-08-03T06:58:51Z","title":"Comparative Validation of GPT-4o-mini and Teacher Mean Scores for Automated Scoring of Music Analysis Responses: Single-Pass Deployment, Repeatability, and Strategy-Specific Bias","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-04T21:00:08.279354Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2608.01783"},"observation_digest":"sha256:9d5aa88c795724eb06d60cca9ed3a20f7097c2ccbe86aa7d5c16843b7d62d950","observation_id":"ff66f522-10ee-44a2-83a0-0e9740e01355","resolution":{"observed_at":"2026-08-04T21:00:08.279354Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2412.12509/citation-record","integrity":"/paper/2412.12509/integrity","json":"/paper/2412.12509/citation-record.json","paper":"/paper/2412.12509"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.070645Z","title":"online\" 'onlinestring :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.070645Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5f5ed27866dd589030417d55ef311ead840714da5dc85a59158e4a620d19bd51","observation_id":"3f65fe95-a1e8-4c60-8f6d-1fc02984aa06","resolution":{"observed_at":"2026-08-11T14:04:26.070645Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.076166Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.076166Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:946f5ee05979c6e8ce11fbe06609f7f35f0dfe2cb95e1f2eaddb5c3a6f53fb42","observation_id":"b1d3aa71-6a72-48bc-b351-afa0ce234c9e","resolution":{"observed_at":"2026-08-11T14:04:26.076166Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0237.2010","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.216227Z","title":null,"venue":null,"work_id":"04b2a925-47f9-4bb8-9ac5-b64322796e10","year":2010},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.081514Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:47ce8d06ebc603d8b62b7b1d04f5c3d7e83bf21ced033f2d524a38820c9bcc20","observation_id":"29d19240-2d42-4122-849d-e5f8cdc2786e","resolution":{"observed_at":"2026-08-11T14:04:28.228940Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.086321Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.086321Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6cac3d4f1000ae9b0dc902c7879912fad81ab942461490178bd4f0afadd12708","observation_id":"4bef4719-b426-4054-937f-bdbb38fcd748","resolution":{"observed_at":"2026-08-11T14:04:26.086321Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/1-4020-3076-2_1","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:03:20.376878Z","title":null,"venue":null,"work_id":"0281d283-44ff-4763-a81b-e87a43926fa7","year":2005},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.090900Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:d36939efacee17ed6e32fefceeeb3f8171d712d93fc0b16939b54c2ad7bcedaa","observation_id":"41426fd2-e2b3-4197-b50d-90e2d36d5ff8","resolution":{"observed_at":"2026-08-11T14:04:27.502966Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.095747Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.095747Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:3f13da5d71276988a0660e18c34d8986e848f04bf63d3e47d81c30075d061f32","observation_id":"3e6156c2-f345-41ae-b409-d593b0b58985","resolution":{"observed_at":"2026-08-11T14:04:26.095747Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/w19-8642","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.482706Z","title":null,"venue":null,"work_id":"3c0936cf-504e-45dc-87ff-3c2f0195cbc2","year":2019},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.100071Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:f3e381e58b7deaab1766807bbbc10be3c4836201db2f279bbae56a7d8beda279","observation_id":"c6362d61-155c-4f5e-9f29-290a423d64a7","resolution":{"observed_at":"2026-08-11T14:04:27.487527Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04667","last_updated":"2025-04-02T15:17:02Z","snapshot_observed_at":"2026-08-14T00:16:56.532983Z","submitted_at":"2024-08-06T16:43:35Z","title":"Non-Determinism of \"Deterministic\" LLM Settings","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.04667","snapshot_observed_at":"2026-08-11T14:04:26.104221Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.104221Z"},"links":{"cited_paper":"/paper/2408.04667","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:256c4a3478244f83f6b27676e8d453596a7f31c77f4748a94d4dc38430ece437","observation_id":"1f02719c-6832-491d-a57d-29855bc180eb","resolution":{"observed_at":"2026-08-11T14:04:26.104221Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.109376Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.109376Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:48649e85994910359c2f5bc8b13b3dc4ec93483f2e1354e0718e3caa594ec5de","observation_id":"d1adfb48-cc4c-4e71-80ad-747c2372d03a","resolution":{"observed_at":"2026-08-11T14:04:26.109376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.464876Z","title":null,"venue":null,"work_id":"7831713a-fd4c-42a0-9b38-c50d37286634","year":2008},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.113779Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:563bd1adf885c46622140ac75d02614c1fd0ca95aedb48ad34d38130573d260d","observation_id":"9fde6601-e8d4-4fc3-bcec-7c9938f941ef","resolution":{"observed_at":"2026-08-11T14:04:28.471331Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.7759/cureus.47468","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.450743Z","title":null,"venue":null,"work_id":"b7b41ac4-2ea0-45b3-94c9-d60095b02204","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.118237Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:2e3f520897cd6476e140e31ad0d604b2bb95e6e9df4b3a6de56fd3f77e24f33e","observation_id":"4861a1e4-1e44-48e4-ab92-ba860b165a52","resolution":{"observed_at":"2026-08-11T14:04:27.455835Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07201","last_updated":"2023-08-14T15:13:04Z","snapshot_observed_at":"2026-08-02T01:34:38.978920Z","submitted_at":"2023-08-14T15:13:04Z","title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07201","snapshot_observed_at":"2026-08-11T14:04:26.122473Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.122473Z"},"links":{"cited_paper":"/paper/2308.07201","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:fcdc9c335d7452f8c185464570eb0b2b282e3ff74ad2f5ee245ce89eb0b5d310","observation_id":"80ce023a-ef11-4ca8-8e30-87d9360f1eb9","resolution":{"observed_at":"2026-08-11T14:04:26.122473Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.10072","last_updated":"2023-12-06T23:20:03Z","snapshot_observed_at":"2026-08-14T12:59:02.765497Z","submitted_at":"2023-12-06T23:20:03Z","title":"Assessing the Usability of GutGPT: A Simulation Study of an AI Clinical Decision Support System for Gastrointestinal Bleeding Risk","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.10072","snapshot_observed_at":"2026-08-11T14:04:26.127287Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.127287Z"},"links":{"cited_paper":"/paper/2312.10072","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:fa9807280797beff0b107eb9fc443038b5a1b9ee49965d85a644d3e2aad8d135","observation_id":"c916ac0d-21f9-4e97-8897-5f766b526d9b","resolution":{"observed_at":"2026-08-11T14:04:26.127287Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10669","last_updated":"2024-09-26T03:16:52Z","snapshot_observed_at":"2026-08-13T04:17:35.878300Z","submitted_at":"2024-02-16T13:21:06Z","title":"Humans or LLMs as the Judge? A Study on Judgement Biases","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10669","snapshot_observed_at":"2026-08-11T14:04:26.132660Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.132660Z"},"links":{"cited_paper":"/paper/2402.10669","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:2526473e1a039399ee3e3567354b345cd9aa3053e37f3f2f75f4409ca860e2f1","observation_id":"2054bfaf-0f8b-4756-894b-d4bd25d69240","resolution":{"observed_at":"2026-08-11T14:04:26.132660Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.445286Z","title":"Gonzalez, Ion Stoica, and Eric P","venue":null,"work_id":"23c3bc66-aecf-49be-814d-5a5fc4ec2127","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.138138Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:84f75d21752a328a1c95f8df31d2117d8915350d8370ab3d9f0f206f7b08ca4d","observation_id":"cb96bfe0-9aa6-4acb-bb98-aa98ae1bacc1","resolution":{"observed_at":"2026-08-11T14:04:28.450953Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-11T14:04:26.142910Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.142910Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:e7f3fabeb100385a81a890fda0acf740544652db4b1b9a64d720f6a73b7e6127","observation_id":"94e3c324-8b82-45bd-a795-e27249e5d1f9","resolution":{"observed_at":"2026-08-11T14:04:26.142910Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.15402","last_updated":"2024-06-06T01:58:54Z","snapshot_observed_at":"2026-08-13T10:04:12.610409Z","submitted_at":"2023-09-27T04:53:10Z","title":"Navigate through Enigmatic Labyrinth A Survey of Chain of Thought Reasoning: Advances, Frontiers and Future","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.15402","snapshot_observed_at":"2026-08-11T14:04:26.148041Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.148041Z"},"links":{"cited_paper":"/paper/2309.15402","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:0d96e554f366f0092369cab6881dd66238af7c501e85b6f7cf71123840c25292","observation_id":"99ec95df-747f-4fcc-b3e7-1723081ceda7","resolution":{"observed_at":"2026-08-11T14:04:26.148041Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.426945Z","title":null,"venue":null,"work_id":"e8771ca4-51f4-462b-b7af-fa19cd9feb94","year":2015},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.153434Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5730e3a48ec6eb5a7de78168fd2bc1b6c2b757e3fb9330f9c3acd7c7f4c7d6e5","observation_id":"b8b96f72-bade-49f0-a319-8d6cb5e25295","resolution":{"observed_at":"2026-08-11T14:04:28.432558Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.158305Z","title":null,"venue":null,"work_id":null,"year":1951},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.158305Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:8f3860cc62cb3effd83af7e58afea2b8add5fb6a9b8fd2654b9ac996b8982f73","observation_id":"1ab7738a-0efc-44f9-9493-af9423cedcf6","resolution":{"observed_at":"2026-08-11T14:04:26.158305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.163353Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.163353Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:b37f3b9a08e08824c09d511e20b8697b08201fb1de0fb188ed175179531f59de","observation_id":"1e63ec38-fe52-4dd3-a39a-f29c82bacce5","resolution":{"observed_at":"2026-08-11T14:04:26.163353Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0544.36452","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.012441Z","title":null,"venue":null,"work_id":"f121d99b-7d59-4263-be15-ba3028376429","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.168312Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:0b9a5859af1efc2f602c596c78afb616abad516294c8ef39460d1193732e75ff","observation_id":"6673ea66-7798-4655-bfc0-407cf3d8bd11","resolution":{"observed_at":"2026-08-11T14:04:28.025265Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11657","last_updated":"2024-06-17T15:41:30Z","snapshot_observed_at":"2026-08-15T01:58:53.395366Z","submitted_at":"2024-06-17T15:41:30Z","title":"Can LLM be a Personalized Judge?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11657","snapshot_observed_at":"2026-08-11T14:04:26.172870Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.172870Z"},"links":{"cited_paper":"/paper/2406.11657","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6f6f39e78b091a5b3aa8ce8b953320495c5d6394370ed735fef5758e5e78de37","observation_id":"32ef8beb-ea30-491e-99d8-69db21ba1f55","resolution":{"observed_at":"2026-08-11T14:04:26.172870Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-11T14:04:26.178088Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.178088Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:c226ea4e2d0191644c8e0f5b755f5bda0da4f346ec777081f45575839a43a9a8","observation_id":"6d907131-1e26-4774-9a01-20fc7cba4637","resolution":{"observed_at":"2026-08-11T14:04:26.178088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.410369Z","title":null,"venue":null,"work_id":"2b0941f8-7ebf-4df7-a2ce-7f798d497ebe","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.182994Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:1c536541dc12f98ca73281a21513550c60775e3be3d0ca222a4495db9911e779","observation_id":"1a4802b6-40ca-4f5d-b723-e812421a0c75","resolution":{"observed_at":"2026-08-11T14:04:28.414973Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.187610Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.187610Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:adbef02071b053cf8d40663f136568ad19ae10d22ad87c99460d17ccedde08c7","observation_id":"3ae35a0a-9323-4ea5-939d-15a4463e99c2","resolution":{"observed_at":"2026-08-11T14:04:26.187610Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.00681","last_updated":"2023-11-01T17:42:45Z","snapshot_observed_at":"2026-08-13T05:34:47.988524Z","submitted_at":"2023-11-01T17:42:45Z","title":"Are Large Language Models Reliable Judges? A Study on the Factuality Evaluation Capabilities of LLMs","version":1},"cited_work":{"arxiv_id":"2311.00681","doi":"10.48550/arxiv.2311.00681","metadata_source":"pith","pith_arxiv_id":"2311.00681","snapshot_observed_at":"2026-08-11T18:16:15.004306Z","title":"Are Large Language Models Reliable Judges? A Study on the Factuality Evaluation Capabilities of LLMs","venue":"cs.CL","work_id":"cf4260fc-8c07-4f71-a0f2-e3c344e4dc30","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.191704Z"},"links":{"cited_paper":"/paper/2311.00681","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6ebfde77594c5ea3aa5982131afb067cb5f620cbe0413761a58f8c2ff0ef9298","observation_id":"87bf65d0-6130-4589-9d76-bef9543a5f93","resolution":{"observed_at":"2026-08-11T14:04:27.283261Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.394317Z","title":null,"venue":null,"work_id":"6401292c-e349-413a-9afb-505780b9b9ec","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.196645Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:a3b75ed62fc3a8bd9b89878a08762d0193b0f1f50cdc313dfafd8c348fad5dd5","observation_id":"5c2fe765-0a6b-4ddb-a64d-ffc1ed84a54c","resolution":{"observed_at":"2026-08-11T14:04:28.398876Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14889","last_updated":"2025-05-16T17:00:45Z","snapshot_observed_at":"2026-08-13T04:13:01.145530Z","submitted_at":"2024-02-22T10:46:11Z","title":"COBIAS: Assessing the Contextual Reliability of Bias Benchmarks for Language Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14889","snapshot_observed_at":"2026-08-11T14:04:26.201098Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.201098Z"},"links":{"cited_paper":"/paper/2402.14889","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:bf115eb0d6de36d973bb08e9a84737121efd2ba803d32723cc11293fd48758ae","observation_id":"4a90643b-6192-42ea-907d-561a7dc7dc56","resolution":{"observed_at":"2026-08-11T14:04:26.201098Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.02781","last_updated":"2026-04-09T12:47:33Z","snapshot_observed_at":"2026-08-13T15:51:22.103374Z","submitted_at":"2023-06-05T11:14:18Z","title":"An Automated Survey of Generative Artificial Intelligence: Large Language Models, Architectures, Protocols, and Applications","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.02781","snapshot_observed_at":"2026-08-11T14:04:26.205923Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.205923Z"},"links":{"cited_paper":"/paper/2306.02781","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:2af75317c5eed290a840daa11997aa88580d9f56f5012b3f3afbce2f8851d364","observation_id":"15610268-167c-47da-82a9-a06be1f82aa4","resolution":{"observed_at":"2026-08-11T14:04:26.205923Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15594","last_updated":"2025-10-19T10:32:43Z","snapshot_observed_at":"2026-08-13T06:43:22.011336Z","submitted_at":"2024-11-23T16:03:35Z","title":"A Survey on LLM-as-a-Judge","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15594","snapshot_observed_at":"2026-08-11T14:04:26.210297Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.210297Z"},"links":{"cited_paper":"/paper/2411.15594","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:1e65a7be25898daf70cafa17c0bfb3845fc0f92973712282f20c793e4d61d646","observation_id":"671e9d4c-0a45-4a95-9e0e-be46453e2b4b","resolution":{"observed_at":"2026-08-11T14:04:26.210297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.18216","last_updated":"2024-10-11T19:05:14Z","snapshot_observed_at":"2026-08-13T04:08:23.797779Z","submitted_at":"2024-02-28T10:19:05Z","title":"LLM Task Interference: An Initial Study on the Impact of Task-Switch in Conversational History","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.18216","snapshot_observed_at":"2026-08-11T14:04:26.215060Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.215060Z"},"links":{"cited_paper":"/paper/2402.18216","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:a39931924f0f9704fc51c319f553e7e9b228d00fd2013c13eedb09fdd9ca8e30","observation_id":"3c730290-85a6-4d6d-8377-38ebd4d3cef9","resolution":{"observed_at":"2026-08-11T14:04:26.215060Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.219545Z","title":null,"venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.219545Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:af49e1adc663a8fec780b0b891a7299bc8b7ad1fbff8fb51d7638863c2968402","observation_id":"1b307189-afcc-4d1b-ac38-54c038d2c24c","resolution":{"observed_at":"2026-08-11T14:04:26.219545Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.377616Z","title":null,"venue":null,"work_id":"8e7d42c1-434f-4e59-82ce-7126ee645f42","year":2016},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.224208Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:fb9a21bee8ddc80b558ea6ad331fd20f680de626291a81e66f52572a4a06bb22","observation_id":"da6c8035-5a83-4618-9483-00e210adf2b3","resolution":{"observed_at":"2026-08-11T14:04:28.382703Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"1756.2002","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.896271Z","title":null,"venue":null,"work_id":"5641abf2-7521-49d7-a66b-bacb5325b200","year":2001},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.228463Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:7c29f0a210e2429f6b4667a98d8f9fc3c4ffaf2eb68e286ee4cbc48b228da0c2","observation_id":"ae659450-e93f-4197-b151-e955b7fa210a","resolution":{"observed_at":"2026-08-11T14:04:27.904959Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.232737Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.232737Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:dc78067882367b3a2627895595c3f4920ff393d92feb4030ee8a4724e41e26af","observation_id":"22f1974d-5748-4616-ac23-cca8accc6f90","resolution":{"observed_at":"2026-08-11T14:04:26.232737Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.18370","last_updated":"2024-07-25T20:04:59Z","snapshot_observed_at":"2026-08-12T23:14:34.174205Z","submitted_at":"2024-07-25T20:04:59Z","title":"Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.18370","snapshot_observed_at":"2026-08-11T14:04:26.236817Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.236817Z"},"links":{"cited_paper":"/paper/2407.18370","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:014376d926025b7cfce45afa4f717a3c69254de18c458b81ee1d28206e9655bf","observation_id":"7760dc1e-5d28-40ab-aa8d-f3cd65ee9f3c","resolution":{"observed_at":"2026-08-11T14:04:26.236817Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s00345-023-04749-6","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.150149Z","title":null,"venue":null,"work_id":"5c64854e-1e6e-4e8f-a578-3b9f473a6d8c","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.241768Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:937ff56ecf929b8982613fa21cb8743347c5e22fa5b9dbb77233379ba245c345","observation_id":"a82f31fe-cce2-40bf-83c8-56b117a486a6","resolution":{"observed_at":"2026-08-11T14:04:27.154764Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.03733","last_updated":"2023-12-08T07:04:52Z","snapshot_observed_at":"2026-08-13T05:15:43.987184Z","submitted_at":"2023-11-28T05:44:06Z","title":"Methods to Estimate Large Language Model Confidence","version":2},"cited_work":{"arxiv_id":"2312.03733","doi":"10.48550/arxiv.2312.03733","metadata_source":"pith","pith_arxiv_id":"2312.03733","snapshot_observed_at":"2026-08-11T18:16:15.004306Z","title":"Methods to Estimate Large Language Model Confidence","venue":"cs.CL","work_id":"2e4c366f-150e-4e0c-bbfb-97e5e2f9210d","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.247067Z"},"links":{"cited_paper":"/paper/2312.03733","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:872bb75879b9ad66c8e8aba0c1a9651a37d3ab0346db620a62ae87c844c0450d","observation_id":"75a55387-d9f1-4a56-bac7-87f3cdf0b900","resolution":{"observed_at":"2026-08-11T14:04:27.140094Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.02077","last_updated":"2023-09-05T09:24:48Z","snapshot_observed_at":"2026-08-14T18:54:21.774751Z","submitted_at":"2023-09-05T09:24:48Z","title":"An Automatic Evaluation Framework for Multi-turn Medical Consultations Capabilities of Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.02077","snapshot_observed_at":"2026-08-11T14:04:26.252447Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.252447Z"},"links":{"cited_paper":"/paper/2309.02077","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:04ff8258201af95a266fed94415467df84eabf642b58d119d372c8454e4600f7","observation_id":"ec6c66c8-414b-4f79-ade6-dd1f91edc147","resolution":{"observed_at":"2026-08-11T14:04:26.252447Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.19187","last_updated":"2024-05-20T01:53:36Z","snapshot_observed_at":"2026-08-13T11:29:16.656241Z","submitted_at":"2023-05-30T16:31:26Z","title":"Generating with Confidence: Uncertainty Quantification for Black-box Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.19187","snapshot_observed_at":"2026-08-11T14:04:26.257770Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.257770Z"},"links":{"cited_paper":"/paper/2305.19187","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:4c1594f17ec8a29d1f1dbefe90bfdd77a1e4d5112f3ecf2a4a072f670899d731","observation_id":"d6a6396b-fbfc-4f06-b44d-e5a465b8ec51","resolution":{"observed_at":"2026-08-11T14:04:26.257770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1177/0013164409344548","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.075318Z","title":null,"venue":null,"work_id":"6a8b033a-c678-4f20-a810-46030223b693","year":2010},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.263043Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:0debccebd31eb45d3c91480d7359883a32b8fdfd3bce01453121be7f531bb2e3","observation_id":"3dbe0a34-d9b6-4d16-9108-b80186e3b1d3","resolution":{"observed_at":"2026-08-11T14:04:27.080584Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08147","last_updated":"2023-08-16T04:56:55Z","snapshot_observed_at":"2026-08-13T10:33:54.024001Z","submitted_at":"2023-08-16T04:56:55Z","title":"MDDial: A Multi-turn Differential Diagnosis Dialogue Dataset with Reliability Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08147","snapshot_observed_at":"2026-08-11T14:04:26.268094Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.268094Z"},"links":{"cited_paper":"/paper/2308.08147","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:2ad1df1407e473615f971d68bbf4968c426e171a9feadc4f8583ce865e2d051a","observation_id":"aec1faed-113e-4995-8e1a-edf8d0830b27","resolution":{"observed_at":"2026-08-11T14:04:26.268094Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2022.10036","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.781671Z","title":null,"venue":null,"work_id":"df8c795d-695d-4bcf-bd9e-e6c1b5289f5d","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.273806Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5b5db4b201d9a0d82287eecc31a02e08b07d7c4a8f8af8976d4081c832e8674a","observation_id":"781bfa25-0dd7-4dfe-985f-6c7838c5138a","resolution":{"observed_at":"2026-08-11T14:04:27.795134Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.347564Z","title":null,"venue":null,"work_id":"a5b0b33a-9198-4c15-9a50-4b7cc4e898f4","year":2013},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.279031Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:df6f8a49dbf20452266fb2acaa9ba62a8961383ed02b86511bd47ef8ba20ed88","observation_id":"940d2f36-2541-4bc5-9e0f-6ea756371786","resolution":{"observed_at":"2026-08-11T14:04:28.353351Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1038/s41562-023-01744-0","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.043439Z","title":null,"venue":null,"work_id":"558fbc49-cd16-47a6-845f-6a943cff1601","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.284638Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:a4f7a722900c961dd510899261a3b3e06d006080b6535b38fbfaf51598cb853c","observation_id":"5a74f683-6d3a-4d8a-898b-c067183023d4","resolution":{"observed_at":"2026-08-11T14:04:27.049265Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.290507Z","title":null,"venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.290507Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5385079699e427c7f837aa60f4277002c73943a901ebf3fdefef03021f885801","observation_id":"98a39769-0f69-4cf9-880f-10983920de19","resolution":{"observed_at":"2026-08-11T14:04:26.290507Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.02828","last_updated":"2024-10-17T17:38:59Z","snapshot_observed_at":"2026-08-15T13:33:56.604670Z","submitted_at":"2023-08-05T09:30:33Z","title":"An Empirical Study of the Non-determinism of ChatGPT in Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.02828","snapshot_observed_at":"2026-08-11T14:04:26.296299Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.296299Z"},"links":{"cited_paper":"/paper/2308.02828","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:7099529beb3f6bc84d9ad036c20569f4de859d11d4839aa0feac191b1da07146","observation_id":"c506b427-5351-411f-8261-9cfecc8d43f6","resolution":{"observed_at":"2026-08-11T14:04:26.296299Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.301949Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.301949Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5ee17a53cfc1590659d03eae90715753682808504f8394b2c21701e4d42e5446","observation_id":"4362e961-78ee-4b5b-b255-ab6a5980d808","resolution":{"observed_at":"2026-08-11T14:04:26.301949Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.00492","last_updated":"2024-05-01T12:59:37Z","snapshot_observed_at":"2026-08-13T00:17:40.767342Z","submitted_at":"2024-05-01T12:59:37Z","title":"Is Temperature the Creativity Parameter of Large Language Models?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.00492","snapshot_observed_at":"2026-08-11T14:04:26.306934Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.306934Z"},"links":{"cited_paper":"/paper/2405.00492","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:9196829ca4b2754d6a8161afa700a2b78d0b7c2977bb7811dc0e461dc88ede71","observation_id":"de8a3632-cf28-4f6d-9697-2b3ff880143e","resolution":{"observed_at":"2026-08-11T14:04:26.306934Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.09138","last_updated":"2025-04-29T03:03:10Z","snapshot_observed_at":"2026-08-14T12:58:41.771567Z","submitted_at":"2023-08-17T18:11:33Z","title":"Semantic Consistency for Assuring Reliability of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.09138","snapshot_observed_at":"2026-08-11T14:04:26.313243Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.313243Z"},"links":{"cited_paper":"/paper/2308.09138","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:f9c93626ecefa2dba34107d290294eba2c9c6acd516ece0a5f8e2e251f6de774","observation_id":"e60797a6-bcfb-4b7f-b150-b3c021014c23","resolution":{"observed_at":"2026-08-11T14:04:26.313243Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1606.05250","last_updated":"2016-10-11T02:42:36Z","snapshot_observed_at":"2026-08-15T14:23:50.348670Z","submitted_at":"2016-06-16T16:36:00Z","title":"SQuAD: 100,000+ Questions for Machine Comprehension of Text","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1606.05250","snapshot_observed_at":"2026-08-11T14:04:26.318583Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.318583Z"},"links":{"cited_paper":"/paper/1606.05250","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:9fda062728ce2f0db1d001acffdbac3d5f5cf036f69ca44e20d129e5a7b9481a","observation_id":"3e3ee178-71ad-4913-b670-d03415d4f7aa","resolution":{"observed_at":"2026-08-11T14:04:26.318583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.21203/rs.3.rs-1486359/v1","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.708594Z","title":"o rg Rahnenf \\","venue":null,"work_id":"5c0d4529-8a43-4275-ac85-627d3958b797","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.323443Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5f74299db875997b7559ccdb2758f5836eef457121e644ff840300c92884c7b3","observation_id":"a8e280c4-1b20-4423-8ec6-11e80aa6d3c3","resolution":{"observed_at":"2026-08-11T14:04:26.713676Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.23186","last_updated":"2024-12-13T22:36:52Z","snapshot_observed_at":"2026-08-12T22:11:37.262478Z","submitted_at":"2024-10-30T16:42:04Z","title":"Reliability of Topic Modeling","version":2},"cited_work":{"arxiv_id":"2410.23186","doi":"10.48550/arxiv.2410.23186","metadata_source":"pith","pith_arxiv_id":"2410.23186","snapshot_observed_at":"2026-08-11T18:16:15.004306Z","title":"Reliability of Topic Modeling","venue":"cs.CL","work_id":"afd98fda-aaef-4863-b1b6-e96f2a2c2a86","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.328647Z"},"links":{"cited_paper":"/paper/2410.23186","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:eed9cde817a0cdb41e648e71580e83aae74708aa52f2e26e03c13db7b7033c5a","observation_id":"c74ba101-823b-43c2-b737-d23881e9d5a4","resolution":{"observed_at":"2026-08-11T14:04:26.697854Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1609/aies.v7i1.31728","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.671619Z","title":null,"venue":null,"work_id":"9993c2b6-22e4-432d-a401-89a3f370dd11","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.334731Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:4a99b8ea13bcfcbd386cdd56eb2cf55d8ace09650522c65ed14d23db2fc646a4","observation_id":"d291bcef-2cca-4f2b-9e64-db3d5928605b","resolution":{"observed_at":"2026-08-11T14:04:26.676373Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07635","last_updated":"2023-08-15T08:32:20Z","snapshot_observed_at":"2026-08-13T10:34:25.965936Z","submitted_at":"2023-08-15T08:32:20Z","title":"LLM-Mini-CEX: Automatic Evaluation of Large Language Model for Diagnostic Conversation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07635","snapshot_observed_at":"2026-08-11T14:04:26.339002Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.339002Z"},"links":{"cited_paper":"/paper/2308.07635","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6fff2e4b3ab635ae32972fdae659b2c816e5b7d9dbd3c3f590ac1e0c5552ba9b","observation_id":"c68cc77f-6828-40e6-90be-145bd210fa1c","resolution":{"observed_at":"2026-08-11T14:04:26.339002Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.02047","last_updated":"2024-08-21T01:58:38Z","snapshot_observed_at":"2026-08-13T04:27:52.898472Z","submitted_at":"2024-02-03T05:52:28Z","title":"Calibration and Correctness of Language Models for Code","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.02047","snapshot_observed_at":"2026-08-11T14:04:26.343589Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.343589Z"},"links":{"cited_paper":"/paper/2402.02047","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:bb6565fabdbac0e217170d8f20e43a01bac9ca8246947da5e3f4a355bc5082e7","observation_id":"139d63d6-5156-4b45-8700-4a9b20864a75","resolution":{"observed_at":"2026-08-11T14:04:26.343589Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.4045/tidsskr.22.0112","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.638163Z","title":null,"venue":null,"work_id":"ffecb429-5c81-4935-8f55-81f25090d587","year":2022},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.348039Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:bc320b54df510b74993547bfaf459158817e6713447c9f9c31cd0d31c731b2e8","observation_id":"cdb82ca0-95f9-4a0a-9e58-3d020e6017f4","resolution":{"observed_at":"2026-08-11T14:04:26.643918Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.10168","last_updated":"2024-04-03T00:25:39Z","snapshot_observed_at":"2026-08-13T10:31:03.133336Z","submitted_at":"2023-08-20T05:31:03Z","title":"Head-to-Tail: How Knowledgeable are Large Language Models (LLMs)? A.K.A. Will LLMs Replace Knowledge Graphs?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.10168","snapshot_observed_at":"2026-08-11T14:04:26.352155Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.352155Z"},"links":{"cited_paper":"/paper/2308.10168","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:d207a90a96812bd47a4fac0f6d5b852a0ca06f9874e1db736ec4b1c47d585895","observation_id":"f03fe839-5613-432f-8a38-d672afe20cbb","resolution":{"observed_at":"2026-08-11T14:04:26.352155Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.357383Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.357383Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:8efb118965bfa53d19e8ae2505befd84a19e734208264cf910b7e5437e45f0d3","observation_id":"66531181-c97b-45de-982d-4bf7dd1faef3","resolution":{"observed_at":"2026-08-11T14:04:26.357383Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.09261","last_updated":"2022-10-17T17:08:26Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-10-17T17:08:26Z","title":"Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.09261","snapshot_observed_at":"2026-08-11T14:04:26.362213Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.362213Z"},"links":{"cited_paper":"/paper/2210.09261","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:0e7a8fbd6b295561f362054989c3b68f8acd3c2008a43acd9e221845b7f133e3","observation_id":"63fe2f30-9075-4174-899b-f79055ea0f04","resolution":{"observed_at":"2026-08-11T14:04:26.362213Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.05688","last_updated":"2024-06-09T08:24:17Z","snapshot_observed_at":"2026-08-12T23:47:06.351105Z","submitted_at":"2024-06-09T08:24:17Z","title":"Peer Review as A Multi-Turn and Long-Context Dialogue with Role-Based Interactions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.05688","snapshot_observed_at":"2026-08-11T14:04:26.367830Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.367830Z"},"links":{"cited_paper":"/paper/2406.05688","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:8d86ad262b477a43e90722bbd14789ee0264e3738a100e153d7d1ca1b4063178","observation_id":"22561902-6e91-4aa9-8bfc-90cf28d2f792","resolution":{"observed_at":"2026-08-11T14:04:26.367830Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08295","last_updated":"2024-04-16T12:52:47Z","snapshot_observed_at":"2026-08-03T03:29:01.959523Z","submitted_at":"2024-03-13T06:59:16Z","title":"Gemma: Open Models Based on Gemini Research and Technology","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08295","snapshot_observed_at":"2026-08-11T14:04:26.372895Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.372895Z"},"links":{"cited_paper":"/paper/2403.08295","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:babf95b68eaf15d4141f5afedd54126d6d3c47595efc1f9b2dfd4a48aa81ae3a","observation_id":"22003b49-d8ac-462a-b0a2-2cf8fd2e3865","resolution":{"observed_at":"2026-08-11T14:04:26.372895Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08295","last_updated":"2024-04-16T12:52:47Z","snapshot_observed_at":"2026-08-03T03:29:01.959523Z","submitted_at":"2024-03-13T06:59:16Z","title":"Gemma: Open Models Based on Gemini Research and Technology","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08295","snapshot_observed_at":"2026-08-11T14:04:26.377902Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.377902Z"},"links":{"cited_paper":"/paper/2403.08295","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:3a1b91cd24d0ffc12c93b6edf4e4679eb54c1f0467883500890d3e69aaaa4e2f","observation_id":"9d77c40a-0fe7-4716-b6ef-1813d5f90c6b","resolution":{"observed_at":"2026-08-11T14:04:26.377902Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14975","last_updated":"2023-10-24T04:27:42Z","snapshot_observed_at":"2026-08-13T11:34:39.120541Z","submitted_at":"2023-05-24T10:12:33Z","title":"Just Ask for Calibration: Strategies for Eliciting Calibrated Confidence Scores from Language Models Fine-Tuned with Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14975","snapshot_observed_at":"2026-08-11T14:04:26.382442Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.382442Z"},"links":{"cited_paper":"/paper/2305.14975","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:a2c728e53db11df0d5d25701e58b01eae46b4ace3d90f8862141603e489b58a9","observation_id":"cf62d123-8a6a-45f8-bc30-bad822d73584","resolution":{"observed_at":"2026-08-11T14:04:26.382442Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.331385Z","title":null,"venue":null,"work_id":"0dd2dc86-6ef6-4fce-a733-d00c301b7ce5","year":2017},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.387119Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:26585869f03eeba098bb3b3aecc1a68717ca633bef5afb32eef0a993d071de36","observation_id":"fc2a4333-b396-4e20-a8d9-b810a0a200de","resolution":{"observed_at":"2026-08-11T14:04:28.336522Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2442.2016","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:27.577539Z","title":null,"venue":null,"work_id":"56902784-b76c-4021-9e6e-e307765d18db","year":2016},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.391672Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:dc8e6af3cd5919aa459ef51df72833a8848224d80099ae86815f4238da1fa11c","observation_id":"b18f4ecd-bf24-47b8-9574-3313ab95869c","resolution":{"observed_at":"2026-08-11T14:04:27.585397Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.396580Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.396580Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6c70363349a44fc7d8433a7183f015e522cc44636e1ee3e163e1a30a675f4ace","observation_id":"cf5d044e-83cb-4661-a71e-71100984f6ca","resolution":{"observed_at":"2026-08-11T14:04:26.396580Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-08-14T05:07:24.912137Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-11T14:04:26.401714Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.401714Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:5500e6b517988ea570486e2cca1f824809b61439184588d34516b25385d5eb1c","observation_id":"717fec84-1dc1-4046-a27b-d65922246322","resolution":{"observed_at":"2026-08-11T14:04:26.401714Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.312447Z","title":null,"venue":null,"work_id":"1c899596-5296-434f-9e44-9e654ce3e9bb","year":2022},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.407205Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:234805502c6f0e1e1a43e3d8ba473ae8f3a3b7fca6c8ef0e05e734ff65e6ff1b","observation_id":"6387e8d1-da56-4929-aa61-ee1e91069c33","resolution":{"observed_at":"2026-08-11T14:04:28.319148Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:26.412482Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.412482Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:e2043a45de173b4a274dfee8c7bdece0e7d47cbaea446bfb3be9f2791f6c1c3e","observation_id":"03db526d-ceeb-49ff-8a7e-66ad750c6e8c","resolution":{"observed_at":"2026-08-11T14:04:26.412482Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02736","last_updated":"2024-10-04T03:57:47Z","snapshot_observed_at":"2026-08-01T08:21:19.528254Z","submitted_at":"2024-10-03T17:53:30Z","title":"Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02736","snapshot_observed_at":"2026-08-11T14:04:26.417029Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.417029Z"},"links":{"cited_paper":"/paper/2410.02736","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:b69d1f18e40d25c9c52ebb145a900b69692bf57c3bbae8745e190b25c6da141a","observation_id":"6b8c3565-afbc-41c7-b394-1cc139df6780","resolution":{"observed_at":"2026-08-11T14:04:26.417029Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04472","last_updated":"2024-08-20T12:36:06Z","snapshot_observed_at":"2026-08-12T23:06:33.418853Z","submitted_at":"2024-08-08T14:02:45Z","title":"Can LLMs Beat Humans in Debating? A Dynamic Multi-agent Framework for Competitive Debate","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.04472","snapshot_observed_at":"2026-08-11T14:04:26.421602Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.421602Z"},"links":{"cited_paper":"/paper/2408.04472","citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:39175611f7fb1a02b66e737bbabe5a59f13436a80bc787176479fdd23cc8cf62","observation_id":"48c8704a-dea7-4808-8826-9404f9201b22","resolution":{"observed_at":"2026-08-11T14:04:26.421602Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.294553Z","title":null,"venue":null,"work_id":"c27ffe21-8e00-42f0-9d2f-829d7d5b811d","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.426067Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:ba7b689e26363d3e29d1853acb5f08674450aa3ed8452a8b566b4eac2aea3e87","observation_id":"0cdf915e-f171-4b0c-be54-2b0e2523af36","resolution":{"observed_at":"2026-08-11T14:04:28.300488Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.276715Z","title":null,"venue":null,"work_id":"9ec278f7-e49f-4de0-a9cc-629256d5cd4f","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.430767Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:6447163d35b58fc03f39ea96fae15314ac0272e8dc63da9e87dda05fba6e13ed","observation_id":"9820e8db-02d7-47a9-9c05-d3fa32b2a478","resolution":{"observed_at":"2026-08-11T14:04:28.281852Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.258539Z","title":null,"venue":null,"work_id":"7f8fb5d3-bd1a-44cc-9a7f-c9ffa6bc7b38","year":2024},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.435520Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:7d1adaffe8170c33dd502a0948b3a268c64509b97cef65b23d5b10c9d5d8e7e0","observation_id":"dbe090d6-eaaf-4828-8b46-f8c6891c72be","resolution":{"observed_at":"2026-08-11T14:04:28.264027Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T14:04:28.241385Z","title":null,"venue":null,"work_id":"984f73a7-c41b-4dfc-bcfb-9908c3d651b6","year":2023},"citing_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-11T14:04:26.439792Z"},"links":{"citing_paper":"/paper/2412.12509"},"observation_digest":"sha256:780b14dcb0e1b60df9f185278302370c446a4cf774cf60c45986a3c99bf7f993","observation_id":"3410c44b-ba9f-4869-8e01-1d563eaff84e","resolution":{"observed_at":"2026-08-11T14:04:28.246278Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-15T09:16:39.557680Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge"},"reference_resolution":{"displayed":76,"state_counts":{"malformed_identifier":1,"metadata_mismatch":2,"parse_uncertain":0,"unresolved":57,"verified_exact":15,"verified_fuzzy":1},"total_outbound_references":76},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 15 August 2026, this Paper Citation Record lists 76 of 76 outbound references and 28 inbound Pith citation observations for arXiv:2412.12509."}