{"as_of":"2026-08-08T04:07:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b7065d33f73cd7b56ca1ae9ada2c641a9dbcc64125e2b3419e5b2e5452197bb0","coverage":[{"denominator":38,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":38,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T18:00:21.298350Z","state":"measured"},{"denominator":43,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":43,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":5,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":5,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T03:31:57.025464Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T22:49:00.988765Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-08-03T03:31:57.025464Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2602.07840","last_updated":"2026-06-10T04:03:01Z","snapshot_observed_at":"2026-08-03T03:31:55.760958Z","submitted_at":"2026-02-08T06:42:50Z","title":"SAGE: Scalable AI Governance & Evaluation","version":4},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-03T03:31:57.025464Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2602.07840"},"observation_digest":"sha256:3ff69a63de111a4f13bf2534c4221998ce3a6c2a7beb39936554cba690686afe","observation_id":"cd4fbf59-8fa1-45e0-8e0f-cdaf24cacc6a","resolution":{"observed_at":"2026-08-03T03:31:57.025464Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-06-30T07:39:08.660337Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:c364b5dcd93feb490738dee900509dbf866f0cba68c5721490936072ab39e7d8","observation_id":"ec6795ed-98e8-4102-9ec4-b83d165037c6","resolution":{"observed_at":"2026-06-30T07:44:21.974095Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-03T22:42:40.207711Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:6f931b85f8d8c7d42afb727a89e574de170e3930bd45e0efb700c8ef90f79cdd","observation_id":"0a0bfecf-24fa-4839-b528-ead159cc4826","resolution":{"observed_at":"2026-07-03T22:49:00.991852Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-12T10:57:57.406511Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-12T10:57:57.406511Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:b2d2c8a6dad59e366a5b757af08cac748dbdec8209bf2c289ab6629f79523d77","observation_id":"5c3361bd-f690-4cfc-9754-a09c8b02a848","resolution":{"observed_at":"2026-07-12T10:57:57.406511Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.30840","last_updated":"2026-06-29T19:16:07Z","snapshot_observed_at":"2026-08-06T13:28:24.425801Z","submitted_at":"2026-06-29T19:16:07Z","title":"Contrastive Reflection for Iterative Prompt Optimization","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-01T02:02:22.841488Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.30840"},"observation_digest":"sha256:2e373038ce4f7e1b1d17ec362ffc4bd0bf556fde3327562c24f887d3dbe18fd5","observation_id":"c43ef256-c6fb-4288-9f91-6d797feb36f6","resolution":{"observed_at":"2026-07-01T12:25:44.293921Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2507.09488/citation-record","integrity":"/paper/2507.09488/integrity","json":"/paper/2507.09488/citation-record.json","paper":"/paper/2507.09488"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2405.05600","last_updated":"2024-05-09T07:39:19Z","snapshot_observed_at":"2026-07-06T18:11:57.100591Z","submitted_at":"2024-05-09T07:39:19Z","title":"Can We Use Large Language Models to Fill Relevance Judgment Holes?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.05600","snapshot_observed_at":"2026-08-06T18:00:21.149269Z","title":"Can we use large language models to fill relevance judgment holes? arXiv preprint arXiv:2405.05600, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.149269Z"},"links":{"cited_paper":"/paper/2405.05600","citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:a5197ff0eb03d8f7edfdebd5d547ac0420d61e5aef89f5daa0becb970980523a","observation_id":"4e55de38-3c98-46fa-bc43-2ca872946aeb","resolution":{"observed_at":"2026-08-06T18:00:21.149269Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.713532Z","title":"Llms can be fooled into labelling a document as relevant: best café near me; this paper is perfectly relevant","venue":null,"work_id":"67271a80-a0de-4c1a-b737-c910a73938c9","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.153555Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:40387a096d481133fbba2def734916e71c8db2e7d9b42179f11ee08bfb799271","observation_id":"48a7e36b-3a97-4b99-a9f7-ab394badd637","resolution":{"observed_at":"2026-08-06T18:00:21.716947Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.703353Z","title":null,"venue":null,"work_id":"0b864ca8-0b6d-4e9c-bb32-a45f680ebf65","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.157366Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:0b2b398448d79150d179c3074d2d5b5c4023a788e6aa6a7014ad52519d6a14f0","observation_id":"3295e0ee-6269-446b-9f02-5754c25466e7","resolution":{"observed_at":"2026-08-06T18:00:21.706807Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.693686Z","title":"Barry and Linda Schamber","venue":null,"work_id":"3eb2bab7-36b8-4e74-9b63-27fb3be5e992","year":1998},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.161831Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:63e9057940f2c554fd2e950b11a99ba35c3e52d6343b3a8d13487a8dde4578c5","observation_id":"f386242e-79cf-439c-bbb8-15f8afa5f40d","resolution":{"observed_at":"2026-08-06T18:00:21.696511Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.683588Z","title":"The concept of relevance in IR","venue":null,"work_id":"3dad04d2-3b5b-4da9-87e3-5e723c921db5","year":2003},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.166395Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:7619b4bd58bff71d781a47bf05d9474cf3d4baabe9c6ceb22073f35b218af6fb","observation_id":"0098a147-bcab-4f39-9460-959af3e8bba2","resolution":{"observed_at":"2026-08-06T18:00:21.687135Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.673400Z","title":"Beyond topicality: A two stage view of relevance and the retrieval process","venue":null,"work_id":"a45ef8bb-9d42-4469-84a9-3b7cde276cba","year":1982},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.171127Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:dc91ed5b585845d1b9a6e4d805bdec5db3122148e38b3f00c7b66172e71ece4a","observation_id":"d019da2b-9c61-447b-9461-634af7fb4d10","resolution":{"observed_at":"2026-08-06T18:00:21.676959Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.662899Z","title":null,"venue":null,"work_id":"e18bfce4-6ef6-4bbd-858b-87dae5a64c27","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.175553Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:9be07583c50f3f1d81516c283b540f5f964434fe53ee83d750491fc6e47d085c","observation_id":"8bc23af9-7059-41b9-b8c1-052ce236e32b","resolution":{"observed_at":"2026-08-06T18:00:21.666517Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.651739Z","title":"Dimensions of Relevance","venue":null,"work_id":"f82a35a5-cfba-4043-9abd-b1986333990b","year":2000},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.179318Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:b0e96c194ae915cd77c1d399b0ace976774db8ca15e5bb596ca795eb0e5ee22c","observation_id":"6be0c392-5cdf-4471-af07-11c8d05cf50c","resolution":{"observed_at":"2026-08-06T18:00:21.655170Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.641880Z","title":"Overview of the TREC 2020 deep learning track","venue":null,"work_id":"e3a414ff-eaef-4473-a988-300941fadbf7","year":2020},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.182993Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:f8425c4b878cec836efdfa637fb79a77aa306fd990f06273e2e06dda46e0d1bd","observation_id":"92fa95d9-72ee-450b-be99-12f718c485c7","resolution":{"observed_at":"2026-08-06T18:00:21.645451Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.630415Z","title":"Voorhees","venue":null,"work_id":"f094ddd0-e461-47f4-bc3c-bbb9509aefd6","year":2019},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.186902Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:1a27b5a75ef65cca2557b6bc5e169688c2f2da9bf2e00dc0f2a51e88abc4a0ff","observation_id":"9f9e401e-4b30-46a6-b60e-3e20b6f2f60f","resolution":{"observed_at":"2026-08-06T18:00:21.634551Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.619250Z","title":"A workbench for autograding retrieve/generate systems","venue":null,"work_id":"a39643ec-b4c7-4ea5-9fc2-4a763fcb1b16","year":1963},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.190464Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:07adbd694428d23d18f0b08eac7ae9b21538b953bb2798356b04355a4ef538ef","observation_id":"a75ccc18-3ed1-44e7-af8d-6d4030693cfd","resolution":{"observed_at":"2026-08-06T18:00:21.623238Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.608125Z","title":"Principles and guidelines for the use of llm judges","venue":null,"work_id":"3938c668-1b08-4de2-92d0-98a5ec8c1e1a","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.194551Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:5b5591465c077d38c20eea591d22e3fd11e71ebc6331dcc5d6fe84d7f9aa2e84","observation_id":"3f42af73-244e-48e9-bae0-16c21eb56037","resolution":{"observed_at":"2026-08-06T18:00:21.612033Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.198187Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.198187Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:43332cf0eb2b719ede671589bd1b3b76f02c551ed840632a1e3b0dbeeca2b157","observation_id":"143437d1-a5a2-4b60-ae78-a195df2a83d9","resolution":{"observed_at":"2026-08-06T18:00:21.198187Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.590274Z","title":"EXAM++: LLM-based Answerability Metrics for IR Evaluation","venue":null,"work_id":"6f009ffe-4c27-45d8-94c0-2dddaf110659","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.201897Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:f206db63b738271dd2281c84248904b9cd188958036ee96845f444f17af3752d","observation_id":"42120bb9-7e03-43ae-a938-60ff3c87f7ac","resolution":{"observed_at":"2026-08-06T18:00:21.593988Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.580705Z","title":"Pencils Down! Automatic Rubric-based Eval- uation of Retrieve/Generate Systems","venue":null,"work_id":"63dd1b58-6f05-455e-af16-93d3f446f766","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.205458Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:50a74421b212e1a56d922a1279743e3a483623358f80d7feac1f931397835610","observation_id":"fbfa5b5a-aef5-45dd-aaa5-a076b269dd02","resolution":{"observed_at":"2026-08-06T18:00:21.583974Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.569794Z","title":null,"venue":null,"work_id":"4e14fa93-30f8-4848-bd79-f5bd17a67535","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.209170Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:647ce528a4379d42bd3df5c8fa53383af03ecb3b134280c925c87b07e4126bfe","observation_id":"5f6a52a7-0155-4d6e-965c-b703378c7df0","resolution":{"observed_at":"2026-08-06T18:00:21.574174Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.560594Z","title":"Bruce Croft","venue":null,"work_id":"376aa7b7-d1da-466e-954c-08d451339285","year":2020},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.213347Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:239f2410cfcbe8ecbdb70b67f43b916f0c964446a23248308c0f7d1b75caf2a9","observation_id":"3bb5820a-542c-438f-9689-0f6013afbe0f","resolution":{"observed_at":"2026-08-06T18:00:21.563843Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.551264Z","title":"One-shot labeling for automatic relevance estimation","venue":null,"work_id":"45d666da-1558-4322-a250-df63316d8d2d","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.217661Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:0d515ec81185d2761448b01f9c4abeea87441f13f957cae5486d822a16657547","observation_id":"6bdb910c-a3f0-482d-83e5-1f5a1024a4ed","resolution":{"observed_at":"2026-08-06T18:00:21.554495Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.541471Z","title":"Crisisfacts: building and evaluating crisis timelines","venue":null,"work_id":"77a9bd76-502f-477f-8a3b-35a3da0a477e","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.221586Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:e52ed205311f24af5f9bdf019d8e129e2b209ecd13e268df0d19a63f9bdf5e91","observation_id":"5432c96d-3fdb-4476-ab94-ec26a4e5c93e","resolution":{"observed_at":"2026-08-06T18:00:21.544892Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.530422Z","title":"Query performance prediction using relevance judgments generated by large language models","venue":null,"work_id":"ac79b9e5-0c79-4d47-910f-a0b2b749ec34","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.225750Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:6a0b5f138995df50d3960ffad4e4a597caf5a70dd9da06c0b4210e429ff9e6fa","observation_id":"616b0a1e-3378-443d-8267-79795fb19790","resolution":{"observed_at":"2026-08-06T18:00:21.535121Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.520341Z","title":"Golbus, and Javed A","venue":null,"work_id":"1e6e4a3e-8dc9-46b4-af59-db5421a89152","year":2012},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.230127Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:403150ad463af068cedd1e0667c33ce7f3488573bb435744b3f12caf008705fe","observation_id":"a14ee621-5997-4b67-afc8-2df1765166e3","resolution":{"observed_at":"2026-08-06T18:00:21.523900Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.510406Z","title":"Initial nugget evaluation results for the trec 2024 rag track with the autonuggetizer framework, 2024","venue":null,"work_id":"3348e362-4d71-423b-a7fe-ec6663db64cf","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.234722Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:e88133f96ee9baad63afb98c7c7403b9a411b82e4b0b463f8c361836da54cf6c","observation_id":"107a5f42-2112-4f1d-95ea-028290d05d1d","resolution":{"observed_at":"2026-08-06T18:00:21.514146Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.499872Z","title":"Rahmani, Varsha Ramineni, Nick Craswell, Bhaskar Mitra, and Emine Yilmaz","venue":null,"work_id":"b1104d96-3c55-4e72-ab77-8c3701ea346c","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.239059Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:492756e2533f45293f6064688259647bb5d26d768d0e0a03db51bee9fdf3b3b3","observation_id":"6438008e-e4a7-4f81-92f4-d07e91ad4e6a","resolution":{"observed_at":"2026-08-06T18:00:21.503471Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.488605Z","title":"Rahmani, Clemencia Siro, Mohammad Aliannejadi, Nick Craswell, Charles L","venue":null,"work_id":"21e0635c-ac0b-442b-8378-4d0317425e5d","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.242928Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:f8e5947a97eb6321c1500f8a496f02bd786a1205c708b3c5239fedd10b49b87c","observation_id":"29250cdc-d00d-4f55-9b9f-ddea7d2f741e","resolution":{"observed_at":"2026-08-06T18:00:21.492219Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.477555Z","title":"Rahmani, Emine Yilmaz, Nick Craswell, and Bhaskar Mitra","venue":null,"work_id":"513dc8db-03a4-4b84-9eec-24e87ba400f2","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.246582Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:d86277d5172a030854bd2ecfdb7cadce44a2591285aa96010fd246db6273aec1","observation_id":"ef6bbbdd-2315-4e54-821a-94cccf3ce142","resolution":{"observed_at":"2026-08-06T18:00:21.481141Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.466213Z","title":"Rahmani, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, Paul Thomas, Charles L","venue":null,"work_id":"8ca06376-1dd4-4ca5-b9ea-e4e19ac92111","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.250427Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:2214a175c545ae862cfadd5e5f0d3329b62203a2c70f6cab656330f58700bfb2","observation_id":"cce1a809-3c56-4288-8703-0fb9976eb3cf","resolution":{"observed_at":"2026-08-06T18:00:21.470027Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.454817Z","title":"Swan: A generic framework for auditing textual conversational systems, 2023","venue":null,"work_id":"aebf078f-f35d-4683-a38f-5b7c7ab066e6","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.255514Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:3566da26c532027670a39557740cec4c95cb7ac99f81250277e7f899c2ca0579","observation_id":"538a8fd2-61b5-4b21-a69f-daef79b94928","resolution":{"observed_at":"2026-08-06T18:00:21.458551Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.442475Z","title":"Exam: How to evaluate retrieve-and-generate systems for users who do not (yet) know what they want","venue":null,"work_id":"74d55b11-7e9f-4857-b646-cdb73af59040","year":2021},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.259126Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:1b631af19ed62772dd5548ce0e49f5543d6d3daa9290945a93008d59a2fc2bfc","observation_id":"3b85eb03-9b97-4519-9550-4618a42a0a70","resolution":{"observed_at":"2026-08-06T18:00:21.446632Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.431286Z","title":"Relevance: A review of the literature and a framework for thinking on the notion in information science","venue":null,"work_id":"5af886ba-12dc-4181-b653-3d0ac903a456","year":1915},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.263223Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:e876cd68c12d3da677634084c53e0cf1eb7f6f36a222ad8fd6cb0f94c1e07665","observation_id":"6299d63e-10d7-481f-850f-34288542503b","resolution":{"observed_at":"2026-08-06T18:00:21.435613Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.267029Z","title":"Don’t use LLMs to make relevance judgments.Information Retrieval Research Journal, 1(1):10.54195/irrj.19625, March 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.267029Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:d28e06236e8634c006166958a4fa276969dcdc2635b91bba1a2e02cd930b46d6","observation_id":"e9881564-ccd6-46d2-baf2-2f35c0fc55fd","resolution":{"observed_at":"2026-08-06T18:00:21.267029Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.420951Z","title":"Is ChatGPT good at search? investigating large language models as re-ranking agents","venue":null,"work_id":"b44f8115-6ac0-4cec-9243-04f331b5c923","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.271399Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:b953b0872e0f1be7d78f10114120025a17e398088cd594b294b2ced7325c7870","observation_id":"5e0823fa-27d5-4f2a-a7f0-513413431ce6","resolution":{"observed_at":"2026-08-06T18:00:21.424555Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.410459Z","title":"Large language models can accurately predict searcher preferences","venue":null,"work_id":"5d558d11-66cd-4a9f-8060-32c75258647c","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.274952Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:1f97142127d32bf763e8f88437deac16cd91b1c1e3a4dfd40269434d2145edc5","observation_id":"4d61a694-27ef-44c9-810a-e42925a020a8","resolution":{"observed_at":"2026-08-06T18:00:21.414114Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.398172Z","title":"Together inference api, 2024","venue":null,"work_id":"368a26e7-64ce-4ab9-997e-f320d1d9116d","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.279139Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:fb36966ae259e2ab26995aee35b4ba749ec7d412f50cc4889f6d302a60d5ace6","observation_id":"c2aefca6-61c7-4556-b164-9a227c056da0","resolution":{"observed_at":"2026-08-06T18:00:21.401636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.283508Z","title":"Umbrela: Umbrela is the (open-source reproduction of the) bing relevance assessor, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.283508Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:18216d359cfaa66ac7e358a22ef201f8259df7b78a84f3aa19bc34c6aa801ebb","observation_id":"90748b70-717d-4419-89ef-5e2d9ec030a8","resolution":{"observed_at":"2026-08-06T18:00:21.283508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.379506Z","title":"Le, and Denny Zhou","venue":null,"work_id":"122f2ad9-3366-491b-8506-3e29ebed25aa","year":2022},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.286995Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:48ce5d9c3ebd658b45ea6d7692fed3cd847723054d0bdd70deb1260a2b10598a","observation_id":"0ce41f22-2051-487c-9a9d-4412b58b58a2","resolution":{"observed_at":"2026-08-06T18:00:21.382904Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.367288Z","title":"Relevance judgment: What do information users consider beyond topicality? Journal of the American Society for Information Science & Technology, 57(7):961–973, May 2006","venue":null,"work_id":"91d567f6-64ef-4b34-8f61-cda595246522","year":2006},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.290488Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:7cb96cd3b4b071ec97edbf41155b4cd3b391143157f814305152682f59b8d121","observation_id":"e80025c9-b9b7-4671-9f2d-d34d9e7ef1d8","resolution":{"observed_at":"2026-08-06T18:00:21.372243Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.356635Z","title":null,"venue":null,"work_id":"eb9d1269-be91-4761-9a4b-3d9de816d2b0","year":1941},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.294167Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:5e5a54c86d7ef9cd00969adc38ac76376c951a2809b9c31c9dc4806ee99579f6","observation_id":"038180a4-47f7-4fad-a042-9ab6179fe786","resolution":{"observed_at":"2026-08-06T18:00:21.360604Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.343993Z","title":null,"venue":null,"work_id":"7b8a22eb-0ee9-4449-b7a9-885c105c93d9","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.298350Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:eddd165b796f74aeb1aba6dd94e88d640856a70083dd60e6de35d9c1862b931c","observation_id":"7c063baa-5628-4e05-8ca8-3832eb1fd7bf","resolution":{"observed_at":"2026-08-06T18:00:21.349510Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-07T06:34:17.273281+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","latest_version":1,"primary_category":"cs.IR","snapshot_observed_at":"2026-08-06T17:52:19.242750Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments"},"reference_resolution":{"displayed":38,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":9,"verified_exact":0,"verified_fuzzy":29},"total_outbound_references":38},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 38 of 38 outbound references and 5 inbound Pith citation observations for arXiv:2507.09488."}