{"as_of":"2026-08-07T08:01:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:070e1a1849f45a77fe923705964697ed552e2f213e805901f53f058ef92d15da","coverage":[{"denominator":21,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":21,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T20:11:41.723677Z","state":"measured"},{"denominator":21,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":21,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-07T06:34:17.273281+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2509.08809/citation-record","integrity":"/paper/2509.08809/integrity","json":"/paper/2509.08809/citation-record.json","paper":"/paper/2509.08809"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.593705Z","title":"Language models are few-shot learners.Advances in neural information processing systems, 33:1877–1901,","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.593705Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:c5d0b9a3cd6b0a4d179d30e13d62f33eb2193b6d2abff2a23c747b90006c6bb9","observation_id":"a1155b71-e107-4bbe-aeae-b27a94747926","resolution":{"observed_at":"2026-08-04T20:11:41.593705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.606910Z","title":"Self-teaching prompting for multi-intent learning with limited super- vision","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.606910Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:c9da1b6fc27455745406b919a1bc2a6d99315faab0ee102db7c1f7b4620c7ad7","observation_id":"780f3e7d-73e0-4609-81b2-53a1f4c6ece5","resolution":{"observed_at":"2026-08-04T20:11:41.606910Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.02027","last_updated":"2019-09-04T18:04:56Z","snapshot_observed_at":"2026-08-04T07:50:24.317079Z","submitted_at":"2019-09-04T18:04:56Z","title":"An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.02027","snapshot_observed_at":"2026-08-04T20:11:41.630126Z","title":"An evaluation dataset for intent classification and out-of-scope prediction.arXiv preprint arXiv:1909.02027,","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.630126Z"},"links":{"cited_paper":"/paper/1909.02027","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:f7982a09d5a8c8089c6aaee5857cfffd98c69c39058509c6e1dab0bf6e783593","observation_id":"fbd2193e-118d-4d9c-bd11-fc22c1d5cc00","resolution":{"observed_at":"2026-08-04T20:11:41.630126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.08291","last_updated":"2023-05-15T01:18:23Z","snapshot_observed_at":"2026-07-06T15:27:02.376191Z","submitted_at":"2023-05-15T01:18:23Z","title":"Large Language Model Guided Tree-of-Thought","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.08291","snapshot_observed_at":"2026-08-04T20:11:41.647592Z","title":"Large language model guided tree-of-thought.arXiv preprint arXiv:2305.08291,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.647592Z"},"links":{"cited_paper":"/paper/2305.08291","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:11190ab1e3f3946341bb74ee1ceb09bce0fe31f170c59682a0b43c73b003ba92","observation_id":"ec452bde-1b91-4f30-835a-199a972eb0e2","resolution":{"observed_at":"2026-08-04T20:11:41.647592Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.13446","last_updated":"2024-12-02T20:55:15Z","snapshot_observed_at":"2026-07-06T17:33:08.817655Z","submitted_at":"2024-02-21T00:44:04Z","title":"Large Language Models for Data Annotation and Synthesis: A Survey","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.13446","snapshot_observed_at":"2026-08-04T20:11:41.653118Z","title":"Large language models for data annotation: A survey.arXiv preprint arXiv:2402.13446,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.653118Z"},"links":{"cited_paper":"/paper/2402.13446","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:7ef7dcce397e585ed3c5ebf1fa123d80e69d90c959cc20616d43e60e1bf2d220","observation_id":"92527c59-d016-460c-89fa-24f1b07f8594","resolution":{"observed_at":"2026-08-04T20:11:41.653118Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.05875","last_updated":"2024-04-08T21:15:36Z","snapshot_observed_at":"2026-07-06T17:57:27.510162Z","submitted_at":"2024-04-08T21:15:36Z","title":"CodecLM: Aligning Language Models with Tailored Synthetic Data","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.05875","snapshot_observed_at":"2026-08-04T20:11:41.660463Z","title":"Codeclm: Aligning language models with tailored synthetic data.arXiv preprint arXiv:2404.05875,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.660463Z"},"links":{"cited_paper":"/paper/2404.05875","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:4d97ff0ba1828e73c80862797a7a60361c7d0e03afcde6e869e3d17e18f6e117","observation_id":"8b125de1-3aa7-49cb-8222-c159da1a5b8f","resolution":{"observed_at":"2026-08-04T20:11:41.660463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-04T20:11:41.668846Z","title":"Finetuned language models are zero-shot learners.arXiv preprint arXiv:2109.01652,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.668846Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:0fe3f0e55c904660fd5a556f09b049afd96bb18812700958ad9bda09fc880efb","observation_id":"1f8d1606-cbe6-497b-9bbe-dbc658520440","resolution":{"observed_at":"2026-08-04T20:11:41.668846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.676658Z","title":"Unigen: A unified framework for textual dataset generation using large language models.arXiv preprint arXiv:2406.18966,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.676658Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:6bcce0c8b88680a77ae7f048b5589b2a9bfaff98f97cd6b9341f09b7e45540c1","observation_id":"eaf3db79-79cd-464d-983f-d6e95b491a94","resolution":{"observed_at":"2026-08-04T20:11:41.676658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.13063","last_updated":"2024-03-17T04:38:48Z","snapshot_observed_at":"2026-07-06T15:45:39.649725Z","submitted_at":"2023-06-22T17:31:44Z","title":"Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.13063","snapshot_observed_at":"2026-08-04T20:11:41.683557Z","title":"Can llms express their uncertainty? an empirical evaluation of confidence elicitation in llms.arXiv preprint arXiv:2306.13063,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.683557Z"},"links":{"cited_paper":"/paper/2306.13063","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:f201dbc2d6d3199490301ea8df046feae8573b7f7cab25e4e4c4107684fe302c","observation_id":"2e045291-eb2a-472f-b5bc-c7a70f28a214","resolution":{"observed_at":"2026-08-04T20:11:41.683557Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.03629","last_updated":"2023-03-10T01:00:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-10-06T01:00:32Z","title":"ReAct: Synergizing Reasoning and Acting in Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.03629","snapshot_observed_at":"2026-08-04T20:11:41.691968Z","title":"React: Synergizing reasoning and acting in language models.arXiv preprint arXiv:2210.03629,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.691968Z"},"links":{"cited_paper":"/paper/2210.03629","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:5923f2e27a344ef7bee8d090f5563e5297e9077a51d1077316070f6747ac6d23","observation_id":"338a3504-808b-4418-8baf-c4d8a489ba21","resolution":{"observed_at":"2026-08-04T20:11:41.691968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.07922","last_updated":"2022-10-22T01:32:03Z","snapshot_observed_at":"2026-07-06T12:38:19.702992Z","submitted_at":"2022-02-16T08:18:02Z","title":"ZeroGen: Efficient Zero-shot Learning via Dataset Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.07922","snapshot_observed_at":"2026-08-04T20:11:41.698533Z","title":"Zerogen: Efficient zero-shot learning via dataset generation.arXiv preprint arXiv:2202.07922,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.698533Z"},"links":{"cited_paper":"/paper/2202.07922","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:5125dc104c18d46c33df34d4d616170c072ac3591ead7e4101b3db30f569bbee","observation_id":"30e50ffc-9452-48bb-ac5d-c14f7fac065b","resolution":{"observed_at":"2026-08-04T20:11:41.698533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.711658Z","title":"Clusterllm: Large language models as a guide for text clustering","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.711658Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:21423145e183b2f922344e2daeaed903c66afb04e2c2d5292cac2b521b7e4cfc","observation_id":"07457aea-99d2-4fe8-a7a1-b60ed49e0470","resolution":{"observed_at":"2026-08-04T20:11:41.711658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06730","last_updated":"2024-07-09T23:53:06Z","snapshot_observed_at":"2026-07-06T17:14:53.706063Z","submitted_at":"2024-01-12T18:03:30Z","title":"Relying on the Unreliable: The Impact of Language Models' Reluctance to Express Uncertainty","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06730","snapshot_observed_at":"2026-08-04T20:11:41.717627Z","title":"Relying on the unreliable: The impact of language models’ reluctance to express uncertainty.arXiv preprint arXiv:2401.06730,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.717627Z"},"links":{"cited_paper":"/paper/2401.06730","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:0dbc562eef7d54b65407453c1ec98b8286a1e06242b96f7330240aea6cf51ec7","observation_id":"e8ead15a-d86e-4033-a881-23c70738845c","resolution":{"observed_at":"2026-08-04T20:11:41.717627Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.723677Z","title":"yi denotes the LLM annotation accuracies","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.723677Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:bd64fc24701a99bbd1e83791c1598637bad183445271a3bc8d3369aa4abd2033","observation_id":"7c32b9ed-be89-420c-b367-3de21fab166e","resolution":{"observed_at":"2026-08-04T20:11:41.723677Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.11610","last_updated":"2022-10-25T17:45:17Z","snapshot_observed_at":"2026-07-06T14:08:26.493996Z","submitted_at":"2022-10-20T21:53:54Z","title":"Large Language Models Can Self-Improve","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.11610","snapshot_observed_at":"2026-08-04T20:11:41.623904Z","title":"Large language models can self-improve.arXiv preprint arXiv:2210.11610,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.623904Z"},"links":{"cited_paper":"/paper/2210.11610","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:790bda766d514d0b3926d63d8c02e9d7f425e8dec104adfdb4a8c0f6bbf97ebe","observation_id":"70bfafd6-52d9-4e8e-a797-b91e80301d73","resolution":{"observed_at":"2026-08-04T20:11:41.623904Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2008.09335","last_updated":"2021-01-27T03:36:21Z","snapshot_observed_at":"2026-07-06T09:49:00.420420Z","submitted_at":"2020-08-21T07:02:11Z","title":"MTOP: A Comprehensive Multilingual Task-Oriented Semantic Parsing Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2008.09335","snapshot_observed_at":"2026-08-04T20:11:41.635830Z","title":"Mtop: A comprehensive multilingual task-oriented semantic parsing benchmark.arXiv preprint arXiv:2008.09335,","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.635830Z"},"links":{"cited_paper":"/paper/2008.09335","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:a18f087873df9df7daeeb83a6f5f1f81fe841946e3e6979bb6c6404f2ca56bf8","observation_id":"8ddb5ba5-2b9b-4000-8a18-fbe2b55a60cb","resolution":{"observed_at":"2026-08-04T20:11:41.635830Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2003.04807","last_updated":"2020-03-10T15:33:54Z","snapshot_observed_at":"2026-08-06T01:22:03.305472Z","submitted_at":"2020-03-10T15:33:54Z","title":"Efficient Intent Detection with Dual Sentence Encoders","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2003.04807","snapshot_observed_at":"2026-08-04T20:11:41.601157Z","title":"Efficient intent detection with dual sentence encoders.arXiv preprint arXiv:2003.04807,","venue":null,"work_id":null,"year":2003},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.601157Z"},"links":{"cited_paper":"/paper/2003.04807","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:01be92cbc85f3491368832cbbeaf165ebc863b385835721a5f2073d6b79f74ab","observation_id":"7c8b7f60-0c63-4701-888c-dcdcc7e57049","resolution":{"observed_at":"2026-08-04T20:11:41.601157Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2205.12914","last_updated":"2025-04-06T23:25:50Z","snapshot_observed_at":"2026-08-01T16:37:38.483173Z","submitted_at":"2022-05-25T17:07:25Z","title":"New Intent Discovery with Pre-training and Contrastive Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.12914","snapshot_observed_at":"2026-08-04T20:11:41.704280Z","title":"New intent discovery with pre-training and contrastive learning.arXiv preprint arXiv:2205.12914,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.704280Z"},"links":{"cited_paper":"/paper/2205.12914","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:1a884b9250894908ac4078933f1d8a9805f840997d5c68a4258387cbbd4d01af","observation_id":"4ace878f-ff85-441e-ae45-d2f847d49ad1","resolution":{"observed_at":"2026-08-04T20:11:41.704280Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.07081","last_updated":"2021-09-16T10:52:59Z","snapshot_observed_at":"2026-07-06T10:59:42.027276Z","submitted_at":"2021-04-14T19:06:11Z","title":"TWEAC: Transformer with Extendable QA Agent Classifiers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.07081","snapshot_observed_at":"2026-08-04T20:11:41.612448Z","title":"Tweac: transformer with extendable qa agent classifiers.arXiv preprint arXiv:2104.07081,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.612448Z"},"links":{"cited_paper":"/paper/2104.07081","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:5e14a3562d08aa435fb44a75317f6b21d199241fe10aa3a0164cb3e7c0f80dfe","observation_id":"71e14911-340e-4b2c-bdec-063d2428ef9e","resolution":{"observed_at":"2026-08-04T20:11:41.612448Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.10147","last_updated":"2018-10-26T21:41:46Z","snapshot_observed_at":"2026-07-06T07:10:12.570041Z","submitted_at":"2018-10-24T01:18:08Z","title":"FewRel: A Large-Scale Supervised Few-Shot Relation Classification Dataset with State-of-the-Art Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.10147","snapshot_observed_at":"2026-08-04T20:11:41.618338Z","title":"Fewrel: A large-scale supervised few-shot relation classification dataset with state-of-the-art evaluation","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.618338Z"},"links":{"cited_paper":"/paper/1810.10147","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:55974af0e135e7778341011e817b5c84436c7fd050fc7deed94d73567866ac2d","observation_id":"8d39323f-e64d-429f-abf4-1048cb2d4f7d","resolution":{"observed_at":"2026-08-04T20:11:41.618338Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.641752Z","title":"Graphprompt: Unifying pre-training and downstream tasks for graph neural networks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.641752Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:321180e959c9cab31c4af38736cbf000ea86c73603034ad3faceb1131c6532bf","observation_id":"b97cea59-8b75-4c4d-b098-1dd48215103d","resolution":{"observed_at":"2026-08-04T20:11:41.641752Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-06T18:25:22.045663Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals"},"reference_resolution":{"displayed":21,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":21,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":21},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-07T06:34:17.273281+00:00","source":"crossref"},{"observed_at":"2026-08-07T06:34:11.927384+00:00","source":"retraction_watch"}],"thesis":"As of 7 August 2026, this Paper Citation Record lists 21 of 21 outbound references and 0 inbound Pith citation observations for arXiv:2509.08809."}