{"as_of":"2026-08-08T09:11:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:f64cd15c53b1bed6f5ade2541ab95904d90a9c68634c8d974ea014ece3f60b4d","coverage":[{"denominator":35,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":35,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T15:02:21.839624Z","state":"measured"},{"denominator":35,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":35,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2507.17015/citation-record","integrity":"/paper/2507.17015/integrity","json":"/paper/2507.17015/citation-record.json","paper":"/paper/2507.17015"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-06T15:02:21.677894Z","title":"Bowman, Zac Hatfield-Dodds , Ben Mann, Dario Amodei, Nicholas Joseph, Sam McCandlish, Tom Brown, and Jared Kaplan","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.677894Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:697b7e06f2c112e20cb28ab6712afd62c924ca25bf75d837822f1fc8550bb9fb","observation_id":"4b92e2cb-d69e-4275-a323-e9cd285741d7","resolution":{"observed_at":"2026-08-06T15:02:21.677894Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.18403","last_updated":"2025-06-02T11:31:19Z","snapshot_observed_at":"2026-07-06T18:37:21.973331Z","submitted_at":"2024-06-26T14:56:13Z","title":"LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.18403","snapshot_observed_at":"2026-08-06T15:02:21.680804Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.680804Z"},"links":{"cited_paper":"/paper/2406.18403","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:512c836773be6353b99c5513eaa25f66c67282c8c98de3aecef8b0b2330ace29","observation_id":"7c29cf48-4c00-4b26-80f0-b9c83b4cd5ec","resolution":{"observed_at":"2026-08-06T15:02:21.680804Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10970","last_updated":"2025-08-08T06:36:34Z","snapshot_observed_at":"2026-07-06T20:22:59.218985Z","submitted_at":"2025-01-19T07:09:11Z","title":"The Alternative Annotator Test for LLM-as-a-Judge: How to Statistically Justify Replacing Human Annotators with LLMs","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.10970","snapshot_observed_at":"2026-08-06T15:02:21.683550Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.683550Z"},"links":{"cited_paper":"/paper/2501.10970","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:e8b94a3a8cbe981b0248a30280cf9e3691b175310d8e0b8e8ccb0dfd4397c3ac","observation_id":"0aeb6484-a2ca-438d-bea8-95f5b98646f7","resolution":{"observed_at":"2026-08-06T15:02:21.683550Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07201","last_updated":"2023-08-14T15:13:04Z","snapshot_observed_at":"2026-08-02T01:34:38.978920Z","submitted_at":"2023-08-14T15:13:04Z","title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07201","snapshot_observed_at":"2026-08-06T15:02:21.686092Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.686092Z"},"links":{"cited_paper":"/paper/2308.07201","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:46a2c33ca00622c4c3bbd60b65276ad0ef532460ad382b76689298c1cd73edd2","observation_id":"696328f2-9cca-404c-8a92-1d7326ee8e1f","resolution":{"observed_at":"2026-08-06T15:02:21.686092Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T04:13:14.044604Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-06T15:02:21.688614Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.688614Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:8a4922fea777096f8d13b2d5734120e184d0b833158abfddd610ba5f2ac4d04f","observation_id":"afa20a44-b53b-4b25-aa8f-c86217d602f9","resolution":{"observed_at":"2026-08-06T15:02:21.688614Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-06T15:02:21.691005Z","title":"Gonzalez, and Ion Stoica","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.691005Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:2a5013e6ab6a8a94a6953d2461d19af29c036b52cb4470f1907357a94522243d","observation_id":"6dbdedf3-427b-4f63-8e0a-0a713190021e","resolution":{"observed_at":"2026-08-06T15:02:21.691005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-06T15:02:21.695714Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.695714Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:849c88882509b3c9c47a8a4b9de1b5fac26314a46de60319c1b1d77606817c02","observation_id":"dff28471-97c9-4e20-ad21-c8e534565395","resolution":{"observed_at":"2026-08-06T15:02:21.695714Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04475","last_updated":"2025-03-10T09:27:03Z","snapshot_observed_at":"2026-07-06T17:56:23.317089Z","submitted_at":"2024-04-06T02:29:02Z","title":"Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.04475","snapshot_observed_at":"2026-08-06T15:02:21.697698Z","title":"Hashimoto","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.697698Z"},"links":{"cited_paper":"/paper/2404.04475","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:e6a39bb47cccb3505c232f49ab8d728dc27d0c52fa3080641fd8640b74fff59e","observation_id":"37dccd96-8a34-4014-9f69-893d509bccae","resolution":{"observed_at":"2026-08-06T15:02:21.697698Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T15:02:22.052636Z","title":"Liang, and Tatsunori B","venue":null,"work_id":"16291acb-64bc-494f-a321-5cb6723b79ca","year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.699949Z"},"links":{"citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:5ee2645f0370b972cde359b071fabaf2172340bcf0b10f29a4dd6d9300c32804","observation_id":"1d85940f-12fb-4336-ad9b-4328bb595a0b","resolution":{"observed_at":"2026-08-06T15:02:22.054590Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.08726","last_updated":"2023-05-31T17:55:02Z","snapshot_observed_at":"2026-08-01T08:54:20.473084Z","submitted_at":"2022-10-17T03:44:30Z","title":"RARR: Researching and Revising What Language Models Say, Using Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.08726","snapshot_observed_at":"2026-08-06T15:02:21.701968Z","title":"Zhao, Ni Lao, Hongrae Lee, Da-Cheng Juan, and Kelvin Guu","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.701968Z"},"links":{"cited_paper":"/paper/2210.08726","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:0ddf837ff54dce7c835febc30809865ec2754c1719a4fafde4588a9cdd3fbecc","observation_id":"203ce50a-051b-4620-9a07-382191bbfa13","resolution":{"observed_at":"2026-08-06T15:02:21.701968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T15:02:22.046677Z","title":null,"venue":null,"work_id":"658c88fa-2678-4f09-8c7d-6f8ca384d6f7","year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.704177Z"},"links":{"citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:cd6e729e87f7b027cb9130d26e99c224242595c23d96a52899e5040fd7be969c","observation_id":"d0c9a754-01be-4817-b58d-b83e65452dd9","resolution":{"observed_at":"2026-08-06T15:02:22.048575Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-04T23:13:25.514661Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-06T15:02:21.706245Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.706245Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:82cc672128b88c301025e8732e839193cb27f7acb73968e2167f33dd6cf3e587","observation_id":"085a6dbf-5564-4730-9830-9f82813a9273","resolution":{"observed_at":"2026-08-06T15:02:21.706245Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16349","last_updated":"2024-01-16T16:51:33Z","snapshot_observed_at":"2026-07-06T16:24:53.814969Z","submitted_at":"2023-09-28T11:18:20Z","title":"Human Feedback is not Gold Standard","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16349","snapshot_observed_at":"2026-08-06T15:02:21.708169Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.708169Z"},"links":{"cited_paper":"/paper/2309.16349","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:a055b298e74b2dc3b5bc556a0ab674349619fd09a2950ad556a69a9943adf01c","observation_id":"c07ab589-2337-441c-99f3-1e134e3650a2","resolution":{"observed_at":"2026-08-06T15:02:21.708169Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.01729","last_updated":"2024-10-02T16:39:58Z","snapshot_observed_at":"2026-07-06T19:26:22.646942Z","submitted_at":"2024-10-02T16:39:58Z","title":"Evaluating Robustness of Reward Models for Mathematical Reasoning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.01729","snapshot_observed_at":"2026-08-06T15:02:21.710305Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.710305Z"},"links":{"cited_paper":"/paper/2410.01729","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:26fe0e0c3fe0cdd5a669a064557c07cbd3128752d25d48267ee500c24f00646b","observation_id":"f0871de8-dcc8-40d7-b89e-cd471039a507","resolution":{"observed_at":"2026-08-06T15:02:21.710305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.16019","last_updated":"2024-12-03T16:18:10Z","snapshot_observed_at":"2026-08-02T20:55:07.319348Z","submitted_at":"2024-04-24T17:51:36Z","title":"The PRISM Alignment Dataset: What Participatory, Representative and Individualised Human Feedback Reveals About the Subjective and Multicultural Alignment of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.16019","snapshot_observed_at":"2026-08-06T15:02:21.712413Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.712413Z"},"links":{"cited_paper":"/paper/2404.16019","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:180c8a923e6fe027ad5d2e3914d80064424e4a075a80aca51a9624cce8957f00","observation_id":"0b126402-104c-43ad-8713-555261521a8b","resolution":{"observed_at":"2026-08-06T15:02:21.712413Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.13787","last_updated":"2024-06-08T16:40:12Z","snapshot_observed_at":"2026-08-02T18:11:57.036767Z","submitted_at":"2024-03-20T17:49:54Z","title":"RewardBench: Evaluating Reward Models for Language Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.13787","snapshot_observed_at":"2026-08-06T15:02:21.714439Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.714439Z"},"links":{"cited_paper":"/paper/2403.13787","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:326aa953ce06e5209f5e7e9f57e6c2efb88276511874de701b0aa3988a2bf25e","observation_id":"fc74ba33-dc17-429c-814b-539bd0c51aea","resolution":{"observed_at":"2026-08-06T15:02:21.714439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.00267","last_updated":"2024-09-03T14:01:54Z","snapshot_observed_at":"2026-07-06T16:13:07.384791Z","submitted_at":"2023-09-01T05:53:33Z","title":"RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.00267","snapshot_observed_at":"2026-08-06T15:02:21.716581Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.716581Z"},"links":{"cited_paper":"/paper/2309.00267","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:a51633977a30d35cd15d73a2c73ac74138db85568871d6eadbe6c33b755227cf","observation_id":"c71181a0-27cd-4845-8b1f-fd2b38befa45","resolution":{"observed_at":"2026-08-06T15:02:21.716581Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.01045","last_updated":"2024-02-11T16:58:02Z","snapshot_observed_at":"2026-08-06T10:54:14.560665Z","submitted_at":"2023-10-02T09:47:40Z","title":"Tool-Augmented Reward Modeling","version":2},"cited_work":{"arxiv_id":"2310.01045","doi":"10.48550/arxiv.2310.01045","metadata_source":"pith","pith_arxiv_id":"2310.01045","snapshot_observed_at":"2026-08-06T18:16:30.548575Z","title":"Tool-Augmented Reward Modeling","venue":"cs.CL","work_id":"7c98bf36-dbdc-44a0-9aeb-02e830547279","year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.718710Z"},"links":{"cited_paper":"/paper/2310.01045","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:dda22f2685c2c65be358f96d8bf567b32c05b525085242f1a09b5aaecbc42809","observation_id":"02a25f2e-4576-4db9-b428-263a8a3f826c","resolution":{"observed_at":"2026-08-06T15:02:21.923606Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11939","last_updated":"2024-10-14T18:11:58Z","snapshot_observed_at":"2026-08-02T11:33:41.297984Z","submitted_at":"2024-06-17T17:26:10Z","title":"From Crowdsourced Data to High-Quality Benchmarks: Arena-Hard and BenchBuilder Pipeline","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11939","snapshot_observed_at":"2026-08-06T15:02:21.721016Z","title":"Gonzalez, and Ion Stoica","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.721016Z"},"links":{"cited_paper":"/paper/2406.11939","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:d59e41e331c636fc029238bd916ee0e5ea556c797f1c1e058e556df2311d426b","observation_id":"2fcb3709-63c9-413c-8b21-1428d29b4c64","resolution":{"observed_at":"2026-08-06T15:02:21.721016Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.20050","last_updated":"2023-05-31T17:24:00Z","snapshot_observed_at":"2026-08-05T13:11:04.104454Z","submitted_at":"2023-05-31T17:24:00Z","title":"Let's Verify Step by Step","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.20050","snapshot_observed_at":"2026-08-06T15:02:21.723067Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.723067Z"},"links":{"cited_paper":"/paper/2305.20050","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:58e8ded2be17529419c17859ffdc52cb74731a911ff913e668db6d9bdca39402","observation_id":"d199b5c7-5a15-4309-a036-690b3351ff3b","resolution":{"observed_at":"2026-08-06T15:02:21.723067Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.07958","last_updated":"2022-05-08T02:43:02Z","snapshot_observed_at":"2026-08-03T16:26:48.747700Z","submitted_at":"2021-09-08T17:15:27Z","title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.07958","snapshot_observed_at":"2026-08-06T15:02:21.725393Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.725393Z"},"links":{"cited_paper":"/paper/2109.07958","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:274ce9eefbb50db2091d5edf2c461c179037e1d869e441cdae3c10e6d6bba835","observation_id":"d14add69-cedc-4091-8fc2-9d9376d3dc99","resolution":{"observed_at":"2026-08-06T15:02:21.725393Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.16634","last_updated":"2023-05-23T22:12:16Z","snapshot_observed_at":"2026-08-02T04:02:36.848064Z","submitted_at":"2023-03-29T12:46:54Z","title":"G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.16634","snapshot_observed_at":"2026-08-06T15:02:21.727560Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.727560Z"},"links":{"cited_paper":"/paper/2303.16634","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:e56c8736653b3451ba1eafb927bc0d807e5b63371e17575438e64fe36957e0ab","observation_id":"3d4ab151-4a94-4f2c-8786-c220d4fce449","resolution":{"observed_at":"2026-08-06T15:02:21.727560Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14251","last_updated":"2023-10-11T05:27:50Z","snapshot_observed_at":"2026-08-01T16:29:58.693560Z","submitted_at":"2023-05-23T17:06:00Z","title":"FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14251","snapshot_observed_at":"2026-08-06T15:02:21.729653Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.729653Z"},"links":{"cited_paper":"/paper/2305.14251","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:245475cf49dcfaa3100d06ffb328358dea627d317a09708c9bad14a232fce5c2","observation_id":"178e20b4-6ba5-470c-8500-9a67a4705c4a","resolution":{"observed_at":"2026-08-06T15:02:21.729653Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T15:02:22.040933Z","title":null,"venue":null,"work_id":"e897d917-0585-4911-9beb-7a4d2d717cc2","year":2022},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.731763Z"},"links":{"citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:a81dc211c83d5a74d900979fa8b8f8edbbdcaa7f17fed8e28ce0ff2882c92c0d","observation_id":"eb1d00de-c94e-4ac2-89c8-b627869962e4","resolution":{"observed_at":"2026-08-06T15:02:22.042880Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.13076","last_updated":"2024-04-15T16:49:59Z","snapshot_observed_at":"2026-07-06T18:02:53.240463Z","submitted_at":"2024-04-15T16:49:59Z","title":"LLM Evaluators Recognize and Favor Their Own Generations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.13076","snapshot_observed_at":"2026-08-06T15:02:21.733580Z","title":"Bowman, and Shi Feng","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.733580Z"},"links":{"cited_paper":"/paper/2404.13076","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:cea2900636c5ed2a65509149c8613e34e9e12d7817ab65cb53027d584eeb8e21","observation_id":"ff84316f-d482-4e36-9d8b-12c0ac14e873","resolution":{"observed_at":"2026-08-06T15:02:21.733580Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.18290","last_updated":"2024-07-29T22:26:36Z","snapshot_observed_at":"2026-08-01T16:34:38.795326Z","submitted_at":"2023-05-29T17:57:46Z","title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.18290","snapshot_observed_at":"2026-08-06T15:02:21.736963Z","title":"Manning, and Chelsea Finn","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.736963Z"},"links":{"cited_paper":"/paper/2305.18290","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:cfc1a0840fedb1cd27872525abdca315e128151915c3ac7666d8424771b6ab88","observation_id":"320efc4c-9de5-4aea-b116-406fdd698f44","resolution":{"observed_at":"2026-08-06T15:02:21.736963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T15:02:22.034892Z","title":"Christiano","venue":null,"work_id":"70775ef1-4401-4d8c-bdec-56c51e129c49","year":2020},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.739234Z"},"links":{"citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:293ef78981c94f76a31a5409251c4b8d707a02c0ee1e63edc734ef5b83a4fc40","observation_id":"b3a45513-346b-4b52-a5cd-055cc172cc3d","resolution":{"observed_at":"2026-08-06T15:02:22.036908Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T15:02:22.028729Z","title":null,"venue":null,"work_id":"a30168cb-11b1-45b1-ade4-df3565bd6c32","year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.741016Z"},"links":{"citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:7585f556b1a11ad99bda58e74f899c43efe2a93a28bbdb3f5ba497d68adee423","observation_id":"4684ed4a-4017-48d3-b2cb-e6924d757c0c","resolution":{"observed_at":"2026-08-06T15:02:22.030841Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18796","last_updated":"2024-05-01T15:37:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-04-29T15:33:23Z","title":"Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18796","snapshot_observed_at":"2026-08-06T15:02:21.742842Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.742842Z"},"links":{"cited_paper":"/paper/2404.18796","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:5f786b337a1c3e07bceece8c614242f14793dcde58199dcc5a75e37eb99ca6cf","observation_id":"5774f7bf-1076-45a0-be24-1b64bd10c7e4","resolution":{"observed_at":"2026-08-06T15:02:21.742842Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09000","last_updated":"2024-04-16T07:41:43Z","snapshot_observed_at":"2026-07-06T16:47:57.524339Z","submitted_at":"2023-11-15T14:41:57Z","title":"Factcheck-Bench: Fine-Grained Evaluation Benchmark for Automatic Fact-checkers","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09000","snapshot_observed_at":"2026-08-06T15:02:21.744854Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.744854Z"},"links":{"cited_paper":"/paper/2311.09000","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:5149c2cb2bf463d55fd68ae341c50efcba646267f20ce7fddeb50468cd10b1e3","observation_id":"fb4bc773-9123-468b-b86b-bc067f993f38","resolution":{"observed_at":"2026-08-06T15:02:21.744854Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.18802","last_updated":"2024-11-07T03:14:38Z","snapshot_observed_at":"2026-08-05T22:52:02.742401Z","submitted_at":"2024-03-27T17:48:55Z","title":"Long-form factuality in large language models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.18802","snapshot_observed_at":"2026-08-06T15:02:21.746960Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.746960Z"},"links":{"cited_paper":"/paper/2403.18802","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:7361e0ab686c778696f818f7fc630be1197a4e0be49ac7c0e83d38c19296a5d8","observation_id":"2fce3b73-9d30-4887-902c-f4726afa870e","resolution":{"observed_at":"2026-08-06T15:02:21.746960Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.03241","last_updated":"2023-09-12T11:01:25Z","snapshot_observed_at":"2026-07-06T16:15:23.878725Z","submitted_at":"2023-09-06T06:18:16Z","title":"GPT Can Solve Mathematical Problems Without a Calculator","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.03241","snapshot_observed_at":"2026-08-06T15:02:21.749110Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.749110Z"},"links":{"cited_paper":"/paper/2309.03241","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:8d87d67563af934e7698070cffd025c9147fd761b841206a8bbfadbf62bb629c","observation_id":"fec041df-7f0e-4951-9943-7ada0188cf39","resolution":{"observed_at":"2026-08-06T15:02:21.749110Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.07641","last_updated":"2024-04-16T04:50:08Z","snapshot_observed_at":"2026-08-06T21:10:28.450711Z","submitted_at":"2023-10-11T16:38:11Z","title":"Evaluating Large Language Models at Evaluating Instruction Following","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.07641","snapshot_observed_at":"2026-08-06T15:02:21.751108Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.751108Z"},"links":{"cited_paper":"/paper/2310.07641","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:32f43abe5b86213b32ed5fb92bd972604931635440a30f8cbe2fe8b5c9b60d55","observation_id":"3cafa3fe-b09b-43de-abd6-af3bd125bee2","resolution":{"observed_at":"2026-08-06T15:02:21.751108Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-06T15:02:21.836882Z","title":"Xing, Hao Zhang, Joseph E","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.836882Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:bb38e9452741e217d38b9d5caa8029adda0409cde9ef949933b5ed9f2d61d5dc","observation_id":"e2f3ce1e-5da4-49ee-80e9-26575453cac5","resolution":{"observed_at":"2026-08-06T15:02:21.836882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.10934","last_updated":"2024-10-16T17:54:12Z","snapshot_observed_at":"2026-08-05T11:14:45.116658Z","submitted_at":"2024-10-14T17:57:02Z","title":"Agent-as-a-Judge: Evaluate Agents with Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.10934","snapshot_observed_at":"2026-08-06T15:02:21.839624Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-06T15:02:21.839624Z"},"links":{"cited_paper":"/paper/2410.10934","citing_paper":"/paper/2507.17015"},"observation_digest":"sha256:0800d7480be2388d994eb4d71d9a6182f38efe922bf0e2a7544537275ea27bba","observation_id":"efc4f697-2a27-46e7-8e10-92199032e735","resolution":{"observed_at":"2026-08-06T15:02:21.839624Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2507.17015","last_updated":"2025-07-22T20:57:09Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T12:17:11.302795Z","submitted_at":"2025-07-22T20:57:09Z","title":"Can External Validation Tools Improve Annotation Quality for LLM-as-a-Judge?"},"reference_resolution":{"displayed":35,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":32,"verified_exact":1,"verified_fuzzy":2},"total_outbound_references":35},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 35 of 35 outbound references and 0 inbound Pith citation observations for arXiv:2507.17015."}