{"as_of":"2026-08-08T14:55:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d5f96932b7539855f3dcd1d6de0dc7883648ff3902af043a33c1a79c7ee2771d","coverage":[{"denominator":35,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":35,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T04:34:13.658964Z","state":"measured"},{"denominator":38,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":38,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-30T15:14:05.529138Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"arxiv_reference","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":0,"observed_at":"2026-08-05T02:28:24.338817Z","source":"arxiv_reference"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":"2506.10403","doi":"10.48550/arxiv.2506.10403","metadata_source":"arxiv_reference","pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Time to impeach LLM -as-a-judge: Programs are the future of evaluation","venue":"ArXiv.org","work_id":"2cf4719c-2fb2-4b3b-aefd-e074a3157869","year":2025},"citing_paper":{"arxiv_id":"2604.04532","last_updated":"2026-07-02T12:30:40Z","snapshot_observed_at":"2026-08-08T10:17:25.391274Z","submitted_at":"2026-04-06T08:54:16Z","title":"Multilingual Prompt Localization for Agent-as-a-Judge: Language and Backbone Sensitivity in Requirement-Level Evaluation","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-05-10T20:02:40.980538Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2604.04532"},"observation_digest":"sha256:e4b8796739fbb086f5c3ad5b61fd9e5646f8b5467e971307bec46ffda3566228","observation_id":"0f017e17-0894-4340-9915-1d2ef361703d","resolution":{"observed_at":"2026-05-10T22:15:50.709981Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":"2506.10403","doi":"10.48550/arxiv.2506.10403","metadata_source":"arxiv_reference","pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Time to impeach LLM -as-a-judge: Programs are the future of evaluation","venue":"ArXiv.org","work_id":"2cf4719c-2fb2-4b3b-aefd-e074a3157869","year":2025},"citing_paper":{"arxiv_id":"2604.05083","last_updated":"2026-04-06T18:36:54Z","snapshot_observed_at":"2026-08-08T12:41:02.824259Z","submitted_at":"2026-04-06T18:36:54Z","title":"Beyond LLM-as-a-Judge: Deterministic Metrics for Multilingual Generative Text Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-10T19:15:37.744461Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2604.05083"},"observation_digest":"sha256:2f015559074f56f6d02d9302d7ea9a1a60e21076439468c142b388bd47b71f72","observation_id":"286f4207-5d0a-40c4-b6ab-05c718a80828","resolution":{"observed_at":"2026-05-10T19:15:44.221307Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-07-30T15:14:05.529138Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.26977","last_updated":"2026-07-29T14:35:29Z","snapshot_observed_at":"2026-08-06T03:59:45.914522Z","submitted_at":"2026-07-29T14:35:29Z","title":"TREK: A Travel Reasoning and Evaluation Kit for LLM Agents in Complex Trip Planning","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-07-30T15:14:05.529138Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2607.26977"},"observation_digest":"sha256:e9a3ec1e7455daa89bb2e7cf3ff7e53196bcd180de20a9ef5201f30527803e03","observation_id":"b7d92186-e247-4ba5-ae91-d64872b60041","resolution":{"observed_at":"2026-07-30T15:14:05.529138Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2506.10403/citation-record","integrity":"/paper/2506.10403/integrity","json":"/paper/2506.10403/citation-record.json","paper":"/paper/2506.10403"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-07-06T19:05:05.530925Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-07T04:34:08.584381Z","title":"Systematic evaluation of llm-as-a-judge in llm alignment tasks: Explainable metrics and diverse prompt templates","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.584381Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:d031b738a0c44d35893a8635a15300c1608ab2dd4b4abc25c6eb9a65ee02cab7","observation_id":"ef315790-9b74-4f37-80c1-6a150431efaa","resolution":{"observed_at":"2026-08-07T04:34:08.584381Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.770228Z","title":"N.; Li, T.; Li, D.; Zhu, B.; Zhang, H.; Jordan, M.; 5 Gonzalez, J","venue":null,"work_id":"118a7b46-a156-4f25-8c2c-926017753486","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.700690Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:165737243e97d9c1244d5cc4840c2897f851e8f2fe27298823c8d393cd4f2ca0","observation_id":"6acd8fde-133d-4d07-9f32-eedf388fa975","resolution":{"observed_at":"2026-08-07T04:34:18.882859Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.569236Z","title":"Advances in Neural Information Processing Systems 2023, 36, 46595–46623","venue":null,"work_id":"2136df04-b880-423c-aa1e-5f1f21f093f8","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.819739Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:f05f5c1358c31d71ba7b9a7bd55ed68ac8f13322222270bfc07e7ae8882aff66","observation_id":"113822d6-e3d9-4b00-a9c9-05185fb377f3","resolution":{"observed_at":"2026-08-07T04:34:18.658458Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.10934","last_updated":"2024-10-16T17:54:12Z","snapshot_observed_at":"2026-08-05T11:14:45.116658Z","submitted_at":"2024-10-14T17:57:02Z","title":"Agent-as-a-Judge: Evaluate Agents with Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.10934","snapshot_observed_at":"2026-08-07T04:34:08.958962Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.958962Z"},"links":{"cited_paper":"/paper/2410.10934","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:1ebd078f842144b1c86e2a1bd7f8035cdf55be7fd4ca8ae9307c31a2a06d3d7d","observation_id":"a029ec1d-50a9-4bef-9736-cd09ca5b1c94","resolution":{"observed_at":"2026-08-07T04:34:08.958962Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.09739","last_updated":"2024-07-17T20:50:11Z","snapshot_observed_at":"2026-07-06T17:30:25.359458Z","submitted_at":"2024-02-15T06:36:07Z","title":"QuRating: Selecting High-Quality Data for Training Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.09739","snapshot_observed_at":"2026-08-07T04:34:09.115749Z","title":"Qurating: Selecting high-quality data for training language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.115749Z"},"links":{"cited_paper":"/paper/2402.09739","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:8a5f4fe4613edc7cb9ce07aea9c2bc357fecd8de2350387e9ac8b4429b853b52","observation_id":"290c994f-6f21-4700-9db7-cf29e9336f1d","resolution":{"observed_at":"2026-08-07T04:34:09.115749Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.351340Z","title":"F.; Leike, J.; Brown, T.; Martic, M.; Legg, S.; Amodei, D","venue":null,"work_id":"1cc1e2c8-7f44-4840-96af-be2848ac8ba3","year":2017},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.307806Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:a7e2e1cebbe2d9b42e170cce5886395845afe63cda4769e65ea13c362882492a","observation_id":"a2b0b47b-a7e8-4a48-85b5-8d71ab66010a","resolution":{"observed_at":"2026-08-07T04:34:18.485017Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.132651Z","title":null,"venue":null,"work_id":"8f769747-2851-44e9-a701-d6df131fd790","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.434801Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:37735d574de8966c9036faf64c9da000adcfdef6f33350b47c8334824d99843a","observation_id":"a74c5cd5-db0d-4001-9307-d494038856ad","resolution":{"observed_at":"2026-08-07T04:34:18.243036Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17631","last_updated":"2025-03-01T17:06:43Z","snapshot_observed_at":"2026-08-06T12:42:08.815092Z","submitted_at":"2023-10-26T17:48:58Z","title":"JudgeLM: Fine-tuned Large Language Models are Scalable Judges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17631","snapshot_observed_at":"2026-08-07T04:34:09.627591Z","title":"Judgelm: Fine-tuned large language models are scalable judges.arXiv preprint arXiv:2310.17631 2023,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.627591Z"},"links":{"cited_paper":"/paper/2310.17631","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:bde61b4e5a3bee93944733b194cc1d5198aa14a48157913b43903c2db20e286e","observation_id":"f980c253-bbe6-46ce-83bc-1e9c8ad4a062","resolution":{"observed_at":"2026-08-07T04:34:09.627591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.11874","last_updated":"2025-02-25T11:04:02Z","snapshot_observed_at":"2026-08-03T12:57:24.468891Z","submitted_at":"2024-05-20T08:30:13Z","title":"xFinder: Large Language Models as Automated Evaluators for Reliable Evaluation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.11874","snapshot_observed_at":"2026-08-07T04:34:09.757567Z","title":"xfinder: Robust and pinpoint answer extraction for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.757567Z"},"links":{"cited_paper":"/paper/2405.11874","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:1c30a92ee689dd15bb2dbc56906f2a8059bb24601eadec5c86e1c9ad12bd7dde","observation_id":"5d5285b6-e5ed-4824-bb79-f08bef70f2f6","resolution":{"observed_at":"2026-08-07T04:34:09.757567Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.03631","last_updated":"2024-06-05T21:29:09Z","snapshot_observed_at":"2026-08-07T22:32:58.170721Z","submitted_at":"2024-06-05T21:29:09Z","title":"Discovering Bias in Latent Space: An Unsupervised Debiasing Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.03631","snapshot_observed_at":"2026-08-07T04:34:09.900260Z","title":"Discovering bias in latent space: An unsupervised debiasing approach","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.900260Z"},"links":{"cited_paper":"/paper/2406.03631","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:6fc0b9bda3dbafc3931e5eec319de8bd5c93352a87b9b51f7a95dbecb5b00126","observation_id":"19dac899-d440-48c8-8b57-7f5e6991fc71","resolution":{"observed_at":"2026-08-07T04:34:09.900260Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.909289Z","title":"H.; Chen, S.; Liu, Z.; Jiang, F.; Wang, B","venue":null,"work_id":"377b1f24-dbf5-4cfe-9bad-390440c377b6","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.023607Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:fd5efbd5e6bd63d67ef557506f3638748e84c381608dd6147ad3c43610f8937c","observation_id":"d5d4530c-185d-4291-8809-faa66fc8ed31","resolution":{"observed_at":"2026-08-07T04:34:18.013754Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02736","last_updated":"2024-10-04T03:57:47Z","snapshot_observed_at":"2026-08-01T08:21:19.528254Z","submitted_at":"2024-10-03T17:53:30Z","title":"Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02736","snapshot_observed_at":"2026-08-07T04:34:10.196137Z","title":"arXiv preprint arXiv:2410.02736 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.196137Z"},"links":{"cited_paper":"/paper/2410.02736","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:dbc240160fa7b02167d7bee77a47e830a7d43aac334ff77ddc7a4e2f656fddec","observation_id":"8f607aac-142f-4e00-b349-0551a68d1b97","resolution":{"observed_at":"2026-08-07T04:34:10.196137Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.649292Z","title":"J.; De Sa, C","venue":null,"work_id":"e68d4ff2-50e2-44c5-918b-659009881cd8","year":2016},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.307887Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:69f9d4160afb50a6e5e5bb43164594414abaa2d323ddcd02e9736634ef833684","observation_id":"c07bf243-e9b2-49b9-9f36-fdb29e17b4f4","resolution":{"observed_at":"2026-08-07T04:34:17.805382Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.437628Z","title":"H.; Ehrenberg, H.; Fries, J.; Wu, S.; Ré, C","venue":null,"work_id":"d1e79aa5-6b5b-4c18-bd97-c66df08d907a","year":2017},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.487327Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:57005874a86b6b5fa5411d6f315aafef4703ece03a1630ef12eae2828f188a31","observation_id":"e8c7f540-ff3c-4302-b88c-0909217293e3","resolution":{"observed_at":"2026-08-07T04:34:17.526067Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.202174Z","title":"Training complex models with multi-task weak supervision","venue":null,"work_id":"fdfb6c58-37dc-44af-95de-3050dd0f9688","year":2019},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.653766Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:da50fc3aba7e9861441729bca700436bedb9bed9114eb0c855390eca60ba8135","observation_id":"ca798256-8249-4e82-8760-09a04738b7a4","resolution":{"observed_at":"2026-08-07T04:34:17.359182Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.042926Z","title":"Fast and three-rious: Speeding up weak supervision with triplet methods","venue":null,"work_id":"edb46c26-845c-487e-b8c3-f2356f4a4b67","year":2020},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.819345Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:1fc6908e8db06d2c6c7b53927cf5193fe2c831c11843247f8764b5d0bd6d19db","observation_id":"affb8cf4-53ab-49ed-bdea-b30ec53d5f5a","resolution":{"observed_at":"2026-08-07T04:34:17.089083Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.13787","last_updated":"2024-06-08T16:40:12Z","snapshot_observed_at":"2026-08-02T18:11:57.036767Z","submitted_at":"2024-03-20T17:49:54Z","title":"RewardBench: Evaluating Reward Models for Language Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.13787","snapshot_observed_at":"2026-08-07T04:34:10.968389Z","title":"Y .; Chandu, K.; Dziri, N.; Kumar, S.; Zick, T.; Choi, Y .; others Rewardbench: Evaluating reward models for language modeling.arXiv preprint arXiv:2403.13787 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.968389Z"},"links":{"cited_paper":"/paper/2403.13787","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:5ce1da97ad5ef7e75bd75a52f40af91ba8469d304bd55f7213e90bfb3b303513","observation_id":"d07bcf6c-d883-4dbd-83f3-8cbd8345e7f8","resolution":{"observed_at":"2026-08-07T04:34:10.968389Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.918586Z","title":"The Twelfth International Conference on Learning Representations","venue":null,"work_id":"fabd5da8-9a09-48cb-88dc-fc41f207fe79","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.147019Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:7a5e0e095b9ab397ee6b35257ce5bfd1ca2f79d8d0bc35144ecb12ec5d9cdace","observation_id":"0e7e9cc8-1810-4351-af7f-ae67533c8332","resolution":{"observed_at":"2026-08-07T04:34:16.956616Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21276","last_updated":"2024-10-25T17:43:01Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-10-25T17:43:01Z","title":"GPT-4o System Card","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21276","snapshot_observed_at":"2026-08-07T04:34:11.322747Z","title":"P.; Perelman, A.; Ramesh, A.; Clark, A.; Ostrow, A.; Welihinda, A.; Hayes, A.; Radford, A.; others Gpt-4o system card","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.322747Z"},"links":{"cited_paper":"/paper/2410.21276","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:c04840ee350e15d2b6f0f3ef975b8ef89a0704e0614f402ffce18625cebb72c9","observation_id":"08063b97-5a06-43dc-b46d-d612811d8fcc","resolution":{"observed_at":"2026-08-07T04:34:11.322747Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.733816Z","title":null,"venue":null,"work_id":"77167843-7a2f-45d1-9c74-01ad916689d0","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.471540Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:888bd1ab9b1834a5fce34fbe47236e56919cdb820b1afd0c3ec1a437922856e4","observation_id":"7849f027-4f10-4708-9357-5ee5d7c19757","resolution":{"observed_at":"2026-08-07T04:34:16.873686Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.443020Z","title":"P.; Fishburne Jr, R","venue":null,"work_id":"d7015ea2-d4cd-472c-b0fc-c9c7001349ad","year":1975},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.632589Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:c9a6f6e85b24b617b2abe34f291b1d3b77b8e2a7feb3cc96973ee940cffbfb51","observation_id":"292117a4-594a-45ad-826b-945007f1d7f5","resolution":{"observed_at":"2026-08-07T04:34:16.587401Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.181056Z","title":"FactKB: Generalizable Factuality Evaluation using Language Models Enhanced with Factual Knowledge","venue":null,"work_id":"1e75fc73-32d4-4fec-b68e-8501673fb244","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.812496Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:b276d08e27c225db22677f397f1c0f14590b51cd68d5d6a8162cd849efeea39e","observation_id":"89546b02-b98c-41dd-a339-c03f10bba31b","resolution":{"observed_at":"2026-08-07T04:34:16.299829Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.829438Z","title":"Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection","venue":null,"work_id":"b47aab82-4d9e-4ff0-b532-c253c4ca59c8","year":2021},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.939726Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:ef87062a22a303eeb8b3a6c87cb41a7aff7e98179dd1007712f678d4cac14a49","observation_id":"8562dfc4-bc17-4ef9-bcd6-9bcc0311d637","resolution":{"observed_at":"2026-08-07T04:34:16.018586Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.541517Z","title":"Universalizing weak supervision","venue":null,"work_id":"141c3c27-27f7-4c76-99ab-bfb310cdf8a9","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.071052Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:7b86975092018e4492473a345f29e2cc69997d96cba0e423ceed6ced00610578","observation_id":"6bdd328f-b03a-4208-8ac1-24a3eca19819","resolution":{"observed_at":"2026-08-07T04:34:15.672364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.225376Z","title":"Weak-to-Strong Generalization Through the Data-Centric Lens","venue":null,"work_id":"6272b063-3e2e-4be7-91f9-376b1c675704","year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.237890Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:fd61dc43f72f4e20d6c8b92119d84ab52e3c34d44a45904bea8f3a342b313001","observation_id":"5711fb54-8843-4ca0-8982-a659fa51253a","resolution":{"observed_at":"2026-08-07T04:34:15.353380Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.907405Z","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","venue":null,"work_id":"2575edf2-e97b-413e-a8fa-f8adb6d4ac85","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.346391Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:6da13ae8a80f9950ab2203e3cdc8d41cd37c475fe245f8b6c0a58a521f9f1a99","observation_id":"f6833448-268e-4a5a-8a56-f5b92c2d2589","resolution":{"observed_at":"2026-08-07T04:34:15.060847Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T04:34:12.492714Z","title":"L.; Almeida, D.; Altenschmidt, J.; Altman, S.; Anadkat, S.; others Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.492714Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:be5eb49666d86f8c7d83b15cbcf9045d5fbabd26ac685d462a29ae9b723bec0b","observation_id":"bb382b29-30e9-41dc-8491-3df9c32420b7","resolution":{"observed_at":"2026-08-07T04:34:12.492714Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-07T04:34:12.651746Z","title":"G.; Hardin, C.; Bhupatiraju, S.; Hussenot, L.; Mesnard, T.; Shahriari, B.; Ramé, A.; others Gemma 2: Improving open language models at a practical size","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.651746Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:56f73c74de0b0c08fb4442e8e72a18e86c5927cb6b8a9f2b62570d36abfef90a","observation_id":"1a7646b5-cff3-46c1-b0d7-07c9a51d592e","resolution":{"observed_at":"2026-08-07T04:34:12.651746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05579","last_updated":"2024-12-10T05:49:12Z","snapshot_observed_at":"2026-07-31T01:42:39.468673Z","submitted_at":"2024-12-07T08:07:24Z","title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05579","snapshot_observed_at":"2026-08-07T04:34:12.759896Z","title":"Llms-as-judges: a comprehensive survey on llm-based evaluation methods","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.759896Z"},"links":{"cited_paper":"/paper/2412.05579","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:44b86a321b5abeeb8ae8c61426bdf84507d4968da2179a4c0208250951f76235","observation_id":"733cb78b-219d-4772-a6d9-9a4b30d992f6","resolution":{"observed_at":"2026-08-07T04:34:12.759896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:12.934810Z","title":"Reference-Guided Verdict: LLMs-as-Judges in Automatic Evaluation of Free-Form Text","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.934810Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:8fa71da38c988580c455c6c18c992007f9bee35872923f420f0628bf845839ef","observation_id":"86e8dac7-4bd3-4ca3-bfa5-b16cfe8053b8","resolution":{"observed_at":"2026-08-07T04:34:12.934810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.730238Z","title":"The ALCHEmist: Automated Labeling 500x CHEaper than LLM Data Annotators","venue":null,"work_id":"5aca053f-0b79-4c8b-99d6-9c0f0a3a3def","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.041107Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:20721554bf0a7f88f45048b8c770733ee07697f751bd86d6dcfd420f3bcfc136","observation_id":"4c2ad085-1fa2-444a-9242-9cd64b26d309","resolution":{"observed_at":"2026-08-07T04:34:14.817946Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.12366","last_updated":"2025-02-17T23:07:14Z","snapshot_observed_at":"2026-08-07T18:10:46.365556Z","submitted_at":"2025-02-17T23:07:14Z","title":"ScriptoriumWS: A Code Generation Assistant for Weak Supervision","version":1},"cited_work":{"arxiv_id":"2502.12366","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.12366","snapshot_observed_at":"2026-08-07T04:34:13.805026Z","title":"ScriptoriumWS: A Code Generation Assistant for Weak Supervision","venue":"cs.LG","work_id":"47e56f7a-f2d5-41c7-bf93-dc7de2d64e19","year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.235686Z"},"links":{"cited_paper":"/paper/2502.12366","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:112b697290e0b4633f2161f14cb9e8ff1ac65a6232be37eb4963fd6a783f076a","observation_id":"abc0dae9-433c-4b1f-96ed-5fa5cfe702e3","resolution":{"observed_at":"2026-08-07T04:34:13.922531Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.444699Z","title":"Autows-bench-101: Benchmarking automated weak supervision with 100 labels","venue":null,"work_id":"0d1051be-95e7-471f-963c-0c4608e36532","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.376326Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:9920cebbfca593d566e5d94ee198fb8c8a62027aa26144df21cd0a797a16432e","observation_id":"5e2df8e1-3ce8-4888-8d33-589c4ba98afa","resolution":{"observed_at":"2026-08-07T04:34:14.616136Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.06708","last_updated":"2026-05-25T22:00:48Z","snapshot_observed_at":"2026-07-06T20:19:50.997046Z","submitted_at":"2025-01-12T04:28:14Z","title":"Evaluating Sample Utility for Efficient Data Selection by Mimicking Model Weights","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.06708","snapshot_observed_at":"2026-08-07T04:34:13.508340Z","title":"Evaluating Sample Utility for Data Selection by Mimicking Model Weights","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.508340Z"},"links":{"cited_paper":"/paper/2501.06708","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:d103360de5d2ac8897af9debe9d333b6b6d6244df60ce38265e6878c469792ca","observation_id":"cc286cef-bb27-4d85-80b2-35448f585fb9","resolution":{"observed_at":"2026-08-07T04:34:13.508340Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.261642Z","title":"\"\"Calculate readability metrics for response","venue":null,"work_id":"0ca0aba9-0783-4bfa-a807-b7f4b20f8779","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.658964Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:45d873a1884d6ce273a64f7d66595e92cd8bf8633d27fea0c15b48ba73500bd7","observation_id":"997eca7d-7afe-4568-bec9-8d04b8b2df7c","resolution":{"observed_at":"2026-08-07T04:34:14.331717Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-07T04:25:03.690277Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation"},"reference_resolution":{"displayed":35,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":16,"verified_exact":1,"verified_fuzzy":18},"total_outbound_references":35},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 35 of 35 outbound references and 3 inbound Pith citation observations for arXiv:2506.10403."}