{"as_of":"2026-08-24T03:12:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:673c7f6e5c65623225d17860275f3670ae10d8b8863602a88080e6024aa06b71","coverage":[{"denominator":140,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-27T20:20:08.996005Z","state":"measured"},{"denominator":100,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":100,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-23T06:30:58.430688+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2606.07936/citation-record","integrity":"/paper/2606.07936/integrity","json":"/paper/2606.07936/citation-record.json","paper":"/paper/2606.07936"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Scientific reports , volume=","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:9ed42ab8aa9d4c1a6d28596dbf4980036eb30bc8151a9d3dc3e664e9a2d48a06","observation_id":"3b16c6cd-27a3-415d-94c2-2b0f64883f7d","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.acl-long.181","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"A Critical Evaluation of Evaluations for Long-form Question Answering","venue":null,"work_id":"35c98394-256d-4dd3-8a3d-ced99533bf02","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:a16778d8c72199036ee1fc71ebb4ce663bb42cad6db9334a461b8209bf761579","observation_id":"98912337-64b8-45d6-be81-e4a388d28567","resolution":{"observed_at":"2026-06-27T20:21:13.088220Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.findings-emnlp.413","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Responsible AI Considerations in Text Summarization Research: A Review of Current Practices","venue":null,"work_id":"74fd609a-a1de-4a11-a017-d623288a2f31","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:171c76f1ad53e0cafd9112bf24104d4fcb811946303b55c0bfe66237fb0adf52","observation_id":"1b430ddb-1507-4fcc-b52a-c5d9be28eed1","resolution":{"observed_at":"2026-06-27T20:21:13.090211Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.acl-long.549","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Automated Metrics for Medical Multi-Document Summarization Disagree with Human Evaluations","venue":null,"work_id":"ffeac45b-392c-4bc1-aa5f-21afdc0f9304","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:f526bae6b50a83fccb292e7d8626f78588b80e4345cdda13f90ef7465aff2c41","observation_id":"2333c820-7649-4aab-9860-e5bd0531c817","resolution":{"observed_at":"2026-06-27T20:21:13.092003Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2025.naacl-demo.44","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"2025 , address=","venue":null,"work_id":"32c8d72f-782b-4ab5-a34f-9e03433186af","year":2025},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:5bf2fbd52b21e270853886f8904189201fc8f43844552effdf3f731ed9d51776","observation_id":"63ca7633-8eed-4921-bf5e-67a5c7b7bb23","resolution":{"observed_at":"2026-06-27T20:21:13.086322Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"First Conference on Language Modeling , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:cb1cf3b7eb7eef004d640ad9742532bc4b2f475d65904a18abc5deb87ad49032","observation_id":"7b62a59c-d62b-40a8-950f-a8e86f33fbef","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM ^2 ) , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:7bd4b929c961e1fd66115698ec76d31d3446a91dd553115c9c4f365c2a66f969","observation_id":"9f402b79-42c9-4c75-bb00-c16d1a5cf483","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"NPJ digital medicine , volume=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:be108e998019503abc0cd64f8a7701f509938a178de314d5cd4a4c714d5a62a8","observation_id":"92e8be16-1bd9-4e99-8479-f931ee0db13d","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:84223b1e22e3404973d103b32d4252b41e3ca09aff6f3bb64db09b7942cca886","observation_id":"7851f010-c71e-4784-819d-5faf2b95d91b","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2nd Workshop on Human Evaluation of NLP Systems (HumEval) , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:b1c631b17e8fe7c0a0009c643b56327e89775df3c169ecd51f9f3ea5f507fe0f","observation_id":"0adf9169-21be-4635-b4f3-e681ab698b64","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the Fourth Workshop on Insights from Negative Results in NLP , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:be166925123a07f36e755eb0e536af1d45bbadf09c3ec5b798488fa8f2b2dc0a","observation_id":"0281dc86-eae2-4c2d-9e48-d6d8d5c9fb87","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Computational Linguistics , volume=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:87882f0ad4f4f5974e2b44edbe29d938fcba0b8b18bdf917313951f1ceab059b","observation_id":"ef034d17-1977-43df-b007-e9d0303743bb","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"University of Chicago Coase-Sandor Institute for Law & Economics Research Paper , number=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:eb6c91eb44fd5be93c163e7bc67ff04dd9709999f920af9f88ad25ee4f052bf3","observation_id":"af3084de-54ff-4635-ab0c-ace829bba0a9","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM ^2 ) , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:1cfd6cc7e87b816a5868daae84b714a043f2089799f6638847e283aa2f25b510","observation_id":"fdea24ff-4e05-4d6a-8772-a5ece13379bd","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2025.acl-short.20","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"LLMs instead of human judges? a large scale empirical study across 20 NLP evaluation tasks","venue":null,"work_id":"40aaa769-7c44-492a-af44-79eb853e4933","year":2025},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:668e972162c042ee2cf46fd0a50bfbff6ece64f3bb574c61a9250292baac82fb","observation_id":"88298a41-01d8-42aa-8d7b-64f7657b6375","resolution":{"observed_at":"2026-06-27T20:21:13.082428Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-07-13T23:49:48.725168+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-13T23:49:48.725168+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Advances in Neural Information Processing Systems , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:4a29f92eae11c3a905803dbe5625ae515059b92639df46244acb696d46e21cc6","observation_id":"4d438264-bbc2-4a75-afce-c3cb7c8db1cf","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"npj Health Systems , volume=","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:896911ebb613d83f279492bdddd70183a47e9f1a6d423a68cb9afd8c76db943a","observation_id":"b7354b67-6b30-476d-8917-49ef0f9f56ea","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2024 ACM Conference on Fairness, Accountability, and Transparency , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:6003e57c7bb423832a1a8a3647a8427fcc51fb5ce339bf238d7faf137c22c925","observation_id":"95367f82-0974-4762-8d4d-4035c3b2d267","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers) , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:2f1fe0e42dd1d005719c0773b956864877fb84bb046b4e0b99258b6bd4116ec4","observation_id":"04d912d3-043c-4af2-b405-f0741be2e3df","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 31st International Conference on Computational Linguistics , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:c1baf7d3e49f51085598b1cd31c9a37fa5aa1bdab31d3f1542b58e15fdfa4110","observation_id":"bcf63442-4f57-444a-9a5a-b5e8ec1ef364","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.05860","last_updated":"2024-05-09T15:48:07Z","snapshot_observed_at":"2026-08-16T18:14:12.748200Z","submitted_at":"2024-05-09T15:48:07Z","title":"The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels","version":1},"cited_work":{"arxiv_id":"2405.05860","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2405.05860","snapshot_observed_at":"2026-07-02T20:37:22.656813Z","title":"The perspectivist paradigm shift: Assumptions and challenges of capturing human labels.arXiv preprint arXiv:2405.05860","venue":null,"work_id":"0dd2ac6f-3fbd-4ec1-9e8c-8ed657241460","year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2405.05860","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:f5000e207bc6af97d23310523017794ee8ecbfda8f224e3ecdf283c3f040a34e","observation_id":"dfe87851-4caa-42ba-b253-e3910c92ec55","resolution":{"observed_at":"2026-07-02T20:37:22.658232Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Nature human behaviour , volume=","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:4b8b5250dcbfedc6cfb6cfd0238fec61ad96847b4bc069dfc55efae771e60171","observation_id":"cd910f03-e810-4dc0-8a36-b4b13e7c6770","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"2022 , url =","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:0269d18eb802242835e8ebedfb1a97a210182f9c8928410937c1b0170fb8f618","observation_id":"31fbb181-f169-4e58-bc82-458b9d881756","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.acl-long.153","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) , pages =","venue":null,"work_id":"52862689-0668-4e00-89e2-e6d5d46e5bb5","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:b6ac377c7fa609294e08bc607d0b521bbdd26aaff4cba4dd8d8a12a444855b9c","observation_id":"01f1039a-74f2-43d3-a22e-a95ece5e03fa","resolution":{"observed_at":"2026-06-27T20:21:13.077833Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:f92b382ec18a845c23d527c0e141cf9d693e170798ab4681d607473c712ed6e0","observation_id":"b523a3cc-2551-4327-bd6d-f8f1b0376a0f","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.07559","last_updated":"2023-12-14T19:40:04Z","snapshot_observed_at":"2026-08-18T17:42:16.299788Z","submitted_at":"2023-12-08T18:50:20Z","title":"PaperQA: Retrieval-Augmented Generative Agent for Scientific Research","version":2},"cited_work":{"arxiv_id":"2312.07559","doi":"10.48550/arxiv.2312.07559","metadata_source":"arxiv_reference","pith_arxiv_id":"2312.07559","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Paperqa: Retrieval-augmented generative agent for scientific research","venue":"arXiv (Cornell University)","work_id":"225eacc5-da70-4566-a2cb-63967c32bb0e","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2312.07559","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:59830463794ceeb6bb5b28d7f63aba6d8fc679963a3ba4515ac8fbb75dbd5164","observation_id":"2bd170a5-8424-4fbd-9a61-b7a86e51a56e","resolution":{"observed_at":"2026-07-02T20:37:22.670917Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-23T01:53:42.089769+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-23T01:53:42.089769+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"arXiv e-prints , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:a4b01f82d79fbd79cf35051cb7ed44599f9b2388a9baf0256d9bb995b0a1fa07","observation_id":"9c9ebfc8-6d43-4280-bf46-7259751c6732","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.14199","last_updated":"2024-11-21T15:07:42Z","snapshot_observed_at":"2026-08-19T08:30:46.152139Z","submitted_at":"2024-11-21T15:07:42Z","title":"OpenScholar: Synthesizing Scientific Literature with Retrieval-augmented LMs","version":1},"cited_work":{"arxiv_id":"2411.14199","doi":"10.48550/arxiv.2411.14199","metadata_source":"arxiv_reference","pith_arxiv_id":"2411.14199","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Weld and Doug Downey and Wen","venue":"arXiv (Cornell University)","work_id":"7b90f8a6-0a1c-48f8-94fc-7f16671e9bae","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2411.14199","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:2bc5ee820e23f0e916cd8083284356b33c41bbea6d29d5a0d8e0c41041aac1f1","observation_id":"ca1cb1e1-32dc-436e-97cc-8d24ac2cce54","resolution":{"observed_at":"2026-07-02T20:37:22.686246Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-25T19:23:21.463275+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-25T19:23:21.463275+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Text summarization branches out , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:5d28e7ba391c263096ec79fa8d424ebd6377acb279818e10d7b35f617ba75508","observation_id":"4ade0dea-bbf2-4462-b04d-0f42eedda8df","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06825","last_updated":"2023-10-10T17:54:58Z","snapshot_observed_at":"2026-08-17T20:30:34.016254Z","submitted_at":"2023-10-10T17:54:58Z","title":"Mistral 7B","version":1},"cited_work":{"arxiv_id":"2310.06825","doi":"10.48550/arxiv.2310.06825","metadata_source":"pith","pith_arxiv_id":"2310.06825","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Mistral 7B","venue":"cs.CL","work_id":"eb5e1305-ad11-4875-ad8d-ad8b8f697599","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2310.06825","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:48d98a62e37ea8313cd585e1516ad7afc62581d8e0d6459fe64e5b312d28bcd1","observation_id":"b5a05ed5-8ba4-4a53-997e-4f5d9c57bedf","resolution":{"observed_at":"2026-07-02T20:37:22.694387Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-10T22:08:12.954417+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-10T22:08:12.954417+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":"2303.08774","doi":"10.1002/tea.20265","metadata_source":"pith","pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"GPT-4 Technical Report","venue":"cs.CL","work_id":"b928e041-6991-4c08-8c81-0359e4097c7b","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:0e02398fe0cdc73d0c1086d77b470c5e98d62ed0df187818fc099d7f6364b99a","observation_id":"17073da2-dec6-4adf-ac80-1ebfb87ec695","resolution":{"observed_at":"2026-07-02T20:37:22.673224Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Clinical Natural Language Processing Workshop , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:b80e589deb8e192e75149acc81f50b10e442f8b15f922f941baf4795078a69d4","observation_id":"3c2823db-2c19-48de-a9f6-3a5156808422","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Conference on Empirical Methods in Natural Language Processing , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:ab437095b1c70cd6b03ed18f72499410502c45e581efc48e834f6477559479db","observation_id":"d7d9876d-33c0-44ff-b2e2-f083aac8379b","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:14c90273f41d761c1c11a59b14c9180c05007d2eb0004994628e969e3c91e3ca","observation_id":"0c7ddf9c-ab5e-44ac-b4e5-623e0f1957c7","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:5f58ca2554a8aebe804a8124930dcff62e5a35900c1cb311c2576255e2cd26f5","observation_id":"7a40d2b5-b97e-49b3-900d-768dd06ffe68","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:3cc2b2c74ba8d85a27ea9e35ae4650d3293b2b0b7295a0a20674a5b81695a32f","observation_id":"d86d1e9b-946b-4b4c-9040-e7ceefc304af","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:f98890398f2393bdd668e0b4f405306110ef9d08431413d1648d4614d140140f","observation_id":"dfb158c3-27fc-47ef-80ac-c778966136db","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":", author=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:68af6836f0ebd2b554d6007daab956262f42eaeee2a063d483b80e5f2159070f","observation_id":"989d5e63-135a-41d1-8d38-37dea9f1b515","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:0117bfb0cdbd54b2eb24add60170cf9ff781c24e5f1cf9679ade5c326fa37917","observation_id":"680c6a2d-57a6-4bd9-8e20-37e1514d6030","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":", author=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:b1a9c8519f72715490702d615fcea12ecef3296791212812493f24223b3eaf8a","observation_id":"01d7754b-ca66-4846-ab93-90daf0fd7095","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":", author=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:47b4098055e9607132caeea43d973b1db72465ae5ec9ea9dcc6a27f5ab8fb650","observation_id":"d82d8d15-5815-49dc-9a3e-dcef00ea0bd7","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3641289","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-10T02:26:42.727008Z","title":"Yu, Qiang Yang, and Xing Xie","venue":"ACM Transactions on Intelligent Systems and Technology","work_id":"baa1ad58-22ae-43a4-86d6-49b6915c12d6","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:e41f08cf997955f77db81efc870a9da6f45cede801423601d6cc53df7e6abb02","observation_id":"79e4abbe-3665-4ae9-b218-ec7115280f91","resolution":{"observed_at":"2026-06-27T20:21:13.063575Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-08T21:08:09.319307+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-08T21:08:09.319307+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.naacl-short.66","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Improving Factuality in Clinical Abstractive Multi-Document Summarization by Guided Continued Pre-training","venue":null,"work_id":"7663d1b9-c02f-4815-a17d-9275b7c1c28f","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:636568384162b941a5b8402b3945004347f8c4e93e357c672638f9161cf11774","observation_id":"79981ea4-a605-417e-bc27-999a4909d88e","resolution":{"observed_at":"2026-06-27T20:21:13.050279Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14251","last_updated":"2023-10-11T05:27:50Z","snapshot_observed_at":"2026-08-19T19:54:57.212265Z","submitted_at":"2023-05-23T17:06:00Z","title":"FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation","version":2},"cited_work":{"arxiv_id":"2305.14251","doi":"10.48550/arxiv.2305.14251","metadata_source":"pith","pith_arxiv_id":"2305.14251","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2305.14251 (2023)","venue":"cs.CL","work_id":"3b4fa1f7-ecc1-4e1a-b434-d8cd24d2ad4c","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2305.14251","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:17f789f5ad54bd58b5376281e45692d89859b6c04f8a37f3d9d2aa0fc4455ebe","observation_id":"3d4ed799-7988-4fa4-af97-67c11843529c","resolution":{"observed_at":"2026-07-02T20:37:22.675805Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.03545","last_updated":"2025-05-31T01:23:14Z","snapshot_observed_at":"2026-08-15T23:39:44.235027Z","submitted_at":"2025-01-07T05:43:23Z","title":"Beyond Factual Accuracy: Evaluating Coverage of Diverse Factual Information in Long-form Text Generation","version":4},"cited_work":{"arxiv_id":"2501.03545","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.03545","snapshot_observed_at":"2026-07-02T20:37:22.687634Z","title":"arXiv preprint arXiv:2501.03545 , year=","venue":null,"work_id":"a360f708-f3eb-4989-8c14-d3c219b849e9","year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2501.03545","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:a037d0ff6dfc580358e0085e967748b1a5e7205dd9d31bec52fce180e637f349","observation_id":"50e44a69-99ad-4685-886a-608f25a6e51c","resolution":{"observed_at":"2026-07-02T20:37:22.689103Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers) , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:ac69ab84c9a385ba690bc0d1ed45eefe8ee0838633ebf52edf48f9aa1912d671","observation_id":"42a97c9f-9026-4ebe-aa8f-a0d5c1ea47b9","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"medRxiv , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:165440b996bfbccb156daa32c00748d3136c37b0e4555be12ee4b7bffd79c67d","observation_id":"8578f1d1-db02-472f-b8a9-4aa23a7a097a","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"and Villarroel, Mauricio and Clifford, Gari D","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:7ac84b2792a263869d40a13dce2aa794ded8493eec197e9ba97eaf4360161ffe","observation_id":"d4fa0a0c-3c0f-48ee-9634-649bcf972adc","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:71f76cbddde8807059f94c1e334f2e7b05c965dc106a2875121d3e80d50d7554","observation_id":"a75cdf03-cbb2-47ce-a62c-5c715c5b1507","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1093/bioinformatics/btz682","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Bioinformatics , volume =","venue":"Bioinformatics","work_id":"e32e1961-872d-4dc5-84f6-be066ccfc32e","year":2019},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:509e4f412893ad98c14a194fea93bc62ca713287f38fff32c0742370c4aee866","observation_id":"6910d7ec-6ce3-40ad-9f75-7ac69f68de76","resolution":{"observed_at":"2026-06-27T20:21:13.065937Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-07-11T03:18:56.980866+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-11T03:18:56.980866+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.naacl-demo.1","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"TOPICAL : TOPIC Pages A utomagica L ly","venue":null,"work_id":"53379d1f-ae61-4e2b-b833-e9b34aa9ba45","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:837f933da89b0f57771be35288a8b66acb8bd88c828f71d63d04e888df5e8dea","observation_id":"e77617e3-a584-442e-8909-e1c024736c16","resolution":{"observed_at":"2026-06-27T20:21:13.059575Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.naacl-main.382","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"What’s in a Summary? Laying the Groundwork for Advances in Hospital-Course Summarization","venue":null,"work_id":"36aa0d14-2241-4184-83c2-74857d4a7482","year":2021},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:3b5091683b323927ef6a1dba110074ce3e78f1979dd371808a17d1eef4d94bb3","observation_id":"500462cd-5edb-4a08-a252-2a017dfee225","resolution":{"observed_at":"2026-06-27T20:21:13.073952Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-25T17:53:27.257618+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-25T17:53:27.257618+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"2015 26th international workshop on database and expert systems applications (dexa) , pages=","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:8cd3ccaba92f863489b2177f7a8a475134b93d2091ad717d930d17a464b8a072","observation_id":"d629c238-c82e-42a1-83b7-272d3e0ee3b7","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Journal of Intelligent Connectivity and Emerging Technologies , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:e9a20e474a5bb1bc067c93adf9c87f2f2dfa4bebcb3e4f77c52c41649edbc5f2","observation_id":"096afb44-0e96-4edd-b81d-1725cbfabe3c","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Journal of biomedical informatics , volume=","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:1bbbbb80325b2af544af579f1dd1e7ce4bdd14cea1dd288c98ad2419d58d648c","observation_id":"6c57ae8e-dbbc-4b85-bf9e-d929a1b47e12","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/w19-1906","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"A Novel System for Extractive Clinical Note Summarization using EHR Data","venue":null,"work_id":"bfacef99-afde-46f4-b4b0-7e333d5fbeef","year":2019},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:06716bd80fb4c8eb16d888c048612818bdffb7f5a0ca900bb54032992a34c631","observation_id":"7222757e-d931-478a-b6d4-35362ae28b33","resolution":{"observed_at":"2026-06-27T20:21:13.042771Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.acl-long.384","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"& Lipton, Z","venue":null,"work_id":"89f40eea-4165-43ab-bd63-320110a5c0bd","year":2021},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:54ac34c348adb83a7b17b3925c4ee22db682e613048d9284acd655082afc6a7d","observation_id":"93100199-bc46-4fef-aed2-dcc1c5576915","resolution":{"observed_at":"2026-06-27T20:21:13.037312Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.nlpmc-1.2","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Towards Automating Medical Scribing : Clinic Visit D ialogue2 N ote Sentence Alignment and Snippet Summarization","venue":null,"work_id":"a2b37183-455a-421b-9d79-67e8f86d236a","year":2021},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:87451c036801cb84683733caf0f6d065945d3b17342dcf710b385c2002e72911","observation_id":"02a5fc08-22d1-44df-bc55-00b3c8ab39dc","resolution":{"observed_at":"2026-06-27T20:21:13.040837Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.clinicalnlp-1.12","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"DERA : Enhancing Large Language Model Completions with Dialog-Enabled Resolving Agents","venue":null,"work_id":"8b2a151c-b7b7-4002-a3ed-665ba8f103c5","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:ffb6312ad43d1dc1681ba5243ba06e2ac2ffff1585aa8d96c040875f083993a9","observation_id":"35044527-a97a-4540-b321-b02590d9c47a","resolution":{"observed_at":"2026-06-27T20:21:13.076027Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"JMIR medical education , volume=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:33f5f350638256bf7645a8718931e483b52265d20eba0812b4ddb2d6fcb865c2","observation_id":"66a7e94b-90ac-4811-b637-e256c3f00507","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Cureus , volume=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:a8906ce7978a8e7bf6f792e3cbad7790a6c5e4884dc45522419d8c3e7577c3aa","observation_id":"3858b38e-2fb3-4e28-b94a-03ee0238e5df","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:11735e0ca2ec0da1c29e9817d4de4d7e8e8ab660bb641572fadb4824a0838300","observation_id":"f1d50d2e-fa1b-4217-9241-dbd2f66e2458","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.09752","last_updated":"2020-11-19T10:03:01Z","snapshot_observed_at":"2026-08-16T19:04:45.086302Z","submitted_at":"2020-11-19T10:03:01Z","title":"From Protocol to Screening: A Hybrid Learning Approach for Technology-Assisted Systematic Literature Reviews","version":1},"cited_work":{"arxiv_id":"2011.09752","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2011.09752","snapshot_observed_at":"2026-07-02T20:37:22.692946Z","title":"CoRR , volume =","venue":null,"work_id":"1b2e68d4-f680-4388-8e4d-ad6077ce1ed8","year":2020},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2011.09752","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:294f366ae171dd434c6d87c02007137dd221462c2e20d80c262448092a1b7a18","observation_id":"83ced66a-83d2-456d-86b9-966a6712b6c2","resolution":{"observed_at":"2026-07-02T20:37:22.694249Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1162/tacl_a_00632","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Hashimoto","venue":"Transactions of the Association for Computational Linguistics","work_id":"0cd9c991-ceb2-4893-821b-2d3de1cad326","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:52eb56f526492d206ee483b9356c41c9974acee728992a67a572ea95acd988e6","observation_id":"499f5a0b-782a-4ff5-bc47-dfb225598d0a","resolution":{"observed_at":"2026-06-27T20:21:13.034125Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-08T21:08:09.375175+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-08T21:08:09.375175+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.naacl-long.478","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"On Learning to Summarize with Large Language Models as References","venue":null,"work_id":"8637dfc0-a4fe-4662-a7e1-40ba84d28712","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:1278cb872727967d861c635cba408269758e78fb49b0280f84d6f38cd47992aa","observation_id":"3d50d6e7-656c-4b2d-b1d7-6fe1267d4be4","resolution":{"observed_at":"2026-06-27T20:21:13.080392Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.acl-short.119","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Summarizing, simplifying, and synthesizing medical evidence using GPT -3 (with varying success)","venue":"Proceedings of the conference - Association for Computational Linguistics. Meeting","work_id":"a0a296f0-67a1-40e9-8b9e-956e94ff41c0","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:f9ba692eba9efbe120fde2303b44fe64ca2feb5de60d5399a4cc0aca31b488f6","observation_id":"d009cb8c-f181-432d-a6d7-9cfca7c6bfc0","resolution":{"observed_at":"2026-06-27T20:21:13.056076Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3545176","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"An empirical survey on long document summarization: Datasets, models, and metrics","venue":"ACM Computing Surveys","work_id":"149054ec-a1b7-45bb-af22-3e1d537aa327","year":2022},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:caa07690b3d8556ff6453bca6cdb16d2a9401deddf81043abe1ac64c287f58b2","observation_id":"d4d46906-9517-447a-802f-13ff88d86416","resolution":{"observed_at":"2026-06-27T20:21:13.048508Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Conference on Empirical Methods in Natural Language Processing , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:78c244fd9125c8023f778356f66a7c3a87de1074613a6f3635b26dca666e9587","observation_id":"f4033730-755b-4562-918d-2a7897fc2b42","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"3912.123391","doi":"10.1145/1233912.1233913","metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"ACM Trans","venue":"ACM Transactions on Speech and Language Processing","work_id":"2be5b4c1-c98a-4f4b-97c0-5612285a6a80","year":2007},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:e5a77e8b4d0dffbfcc0773000398f85d701a4a56fd0cf458fa0102ca7e2a804e","observation_id":"cf87f024-4abc-49d1-928d-8dc81c87a1e7","resolution":{"observed_at":"2026-06-27T20:21:13.073698Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3584700","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"2023 , journal =","venue":"ACM Computing Surveys","work_id":"c6fb6271-0c7f-446b-9de6-1d9d4de89d5c","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:332ca6d8b6751bc9094cb9afe9d3f33cf1139a27e0a1cfb9b919cdaef1aa37f8","observation_id":"c59feabe-6e95-4cd8-955d-553e116ae950","resolution":{"observed_at":"2026-06-27T20:21:13.046915Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:6e404e5077005d0bd953c7f54d12447995656e511642b5df72a451323107a71e","observation_id":"3525dbf9-67e6-4d0f-9ca2-c17c250cca5c","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:718a5de27e23d9de624e707b8a8cabebac7419159e73df34d73efa22e02ca571","observation_id":"058447c0-3117-4e47-bf33-1e40dbf769ef","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Journal of Medical Internet Research , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:be13738b388dd9d6190a4e9dc4f5c7018b98706a3b03acffb2a8c0f032ae9121","observation_id":"df9f31c7-484f-4912-969d-a97d9286169e","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"BMJ Open , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:6164f763f1cf4eca8cdbda06b0dcefd6536ab2f7ccd6ac33332e5b9c29728fb0","observation_id":"23a06557-3ed7-47ad-8984-5345d7388e81","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Energies , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:45afb7e207cfb94339c3c222e8541edabec0f154c60aa8bfe5fd23d8cad5d0cd","observation_id":"e273582a-ea3e-4508-98b3-f36c2d1b034b","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 52nd annual meeting of the association for computational linguistics (volume 1: Long papers) , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:eef3ff9ec91486e472c32e9f92dd9a7001168e2fa99d07b494bc6ad46aa27e32","observation_id":"71b6e608-f57d-4808-904c-a7a0a6935368","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:61b100b0b14bc5fb34aff366b5ec2072a63b22f7268331aba549bc904bb1ec94","observation_id":"29700864-7123-40c8-a28a-0287941ffb0e","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the conference","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:2ae4badf1ba728fb615f94c7d4b499fd92d8971d3b4961ee0f298986cc4c5666","observation_id":"2fdf8532-0405-4246-97b5-f7d5d0c3c282","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Trends in cognitive sciences , volume=","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:d0529f359300348a83e256c6d7efc62bb4a9c524b70b919e748460f20f20edca","observation_id":"a0844e4b-fb9c-4de3-a675-905732047568","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1016/b0-08-043076-7/01508-4","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Craik , abstract =","venue":"Elsevier eBooks","work_id":"8c0578f0-b7e5-4b5a-9668-c70ce7199756","year":2001},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:e90d7968c2a58fe9464a7e9e42da59e4dce6b5b75bcd16e73df65aa1f9feebff","observation_id":"8a318ff8-fdff-459b-aea2-26b4ea2b1a21","resolution":{"observed_at":"2026-06-27T20:21:13.081689Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Conference on Empirical Methods in Natural Language Processing , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":81,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:b218d2313a607ac83a754e026bf627abbdc2f733602299cbea0d48d07ec9e763","observation_id":"bdf59089-fc1d-4dc1-861a-6b7b06fcdbe2","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":82,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:ee1272aac41f7490c0e43263561ca15bd09ea7d81d310f3d377e3123e08c17a5","observation_id":"add8bb77-99b8-44f8-a243-419ffef34766","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":83,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:9e44a63f62c59e1f07e2a023c21905cbbd70307191baeece61c3d9fc427ce34c","observation_id":"b8041b7b-47db-417a-8afc-c4eb9c13c160","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Annual Meeting of the Association for Computational Linguistics , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:4b5dd5defc26fa62b54f13cf34393db0ed9c4cf86bf91f1dfffce7236149f898","observation_id":"40afc49a-429e-48b5-a658-cfab1b4c2c3e","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":85,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:9826d3ed38a989b0eac82df1dd5767af060749fe8fab759ab4dea6e6e27cc49f","observation_id":"19d1e942-015b-4bf7-8162-42ab26d79b24","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"ArXiv , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":86,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:58c4e534492478aa8e52b67100a96387e0acee0fc6b987c708ade11a0d5ec196","observation_id":"ec1320b5-868c-48f4-b46d-4da9355d15af","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.findings-acl.102","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"H i S truct+: Improving Extractive Text Summarization with Hierarchical Structure Information","venue":"Findings of the Association for Computational Linguistics: ACL 2022","work_id":"24ddfad0-c87a-43bf-b944-eea005875d0e","year":2022},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:6b3c760d231a35ffeea8a195e6730ef4624321da9cfd5e29826e51f7dddd6c87","observation_id":"5d914574-9092-439e-9e21-c88e7c18bdad","resolution":{"observed_at":"2026-06-27T20:21:13.027023Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.emnlp-main.543","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"What factors might be causing the significant deviations in my circadian rhythm patterns over the past 30 days?","venue":null,"work_id":"0822f722-ab42-4582-acf4-337a65f25ddd","year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:852d1575e03b9f8a9efcf900dc2af549629432816e2d7a3b646bc97205128699","observation_id":"997af391-06bd-48f0-a7e6-ce3fda19e9c1","resolution":{"observed_at":"2026-06-27T20:21:13.057471Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"and Belz, Anya and Du s ek, Ond r ej and Mille, Simon and van der Lee, Chris and Reiter, Ehud and Santhanam, Shivani and Thomson, Craig","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:1880dc0c54696f07218284510ef43196248e3c2d289552e004eef9e9153aeef8","observation_id":"2418d2f6-894b-4402-835b-8e4857e8bdbc","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"G-Eval : NLG Evaluation using GPT-4 with Better Human Alignment","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":90,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:e7563f871df8469fae25334eec5e8e7f9df62088e33ba9a46f0c9969cf69eb8d","observation_id":"addd4d8d-530d-45cd-8296-1f1f828e1ab5","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.acl-long.413","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"A Scoping Review of LLM-as-a-Judge in Healthcare and the MedJUDGE Governance Framework","venue":null,"work_id":"ceeb9487-46b5-4dcb-baf4-6b6a86323b17","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:6f395809b23a77d57e6c3e39672f173232f091130bfb4f9c4d843668156d8ed5","observation_id":"81dc610d-89e7-45e1-b7e4-cf622802c639","resolution":{"observed_at":"2026-06-27T20:21:13.080084Z","resolver_source":"doi","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-07-11T15:53:23.737646+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-11T15:53:23.737646+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.15754","last_updated":"2024-02-24T08:01:32Z","snapshot_observed_at":"2026-08-16T14:15:43.091584Z","submitted_at":"2024-02-24T08:01:32Z","title":"HD-Eval: Aligning Large Language Model Evaluators Through Hierarchical Criteria Decomposition","version":1},"cited_work":{"arxiv_id":"2402.15754","doi":"10.48550/arxiv.2402.15754","metadata_source":"arxiv_reference","pith_arxiv_id":"2402.15754","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"HD - Eval : Aligning Large Language Model Evaluators Through Hierarchical Criteria Decomposition , February 2024","venue":"arXiv (Cornell University)","work_id":"18e668b5-0860-495a-94d0-654a7076aa08","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":92,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2402.15754","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:dff8aa5a565f9954d0b0467fa6a6692b1351c69fb80626112268d48c5eb36c5d","observation_id":"b8eb6e69-c084-4c87-b914-610b0f4c24ba","resolution":{"observed_at":"2026-07-02T20:37:22.686429Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-05-23T23:53:01.460548+00:00","source":"crossref_status_cache"},{"observed_at":"2026-05-23T23:53:01.460548+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Findings of the Association for Computational Linguistics: EMNLP 2023 , year=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":93,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:20388f0879a6760338cc76120c58e631b4344c4c35bece7bc03b116d9a622f05","observation_id":"2cabe6f7-8e1a-4cda-abb1-132bd67c0256","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP) , year=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":94,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:647e156b4cb4f1beeafef94783319a92e333b8a04a019917f7d644324cb90795","observation_id":"1eac47b5-1fd2-427e-9246-ef72f0dda100","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.naacl-long.33","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)","venue":null,"work_id":"4c9b4d67-8a3b-4195-a768-6e4ba1c4f490","year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":95,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:5d2123c0f6d19dc9935a461d2ed24803ee0310f8dbfea196e5ab1ba700f2398d","observation_id":"d52780b4-b71b-4926-aea5-9ddee4067efc","resolution":{"observed_at":"2026-06-27T20:21:13.024730Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Transactions of the Association for Computational Linguistics , volume=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":96,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:388ccdf9ddc4a3224e1de021471d7aef70add81a833f741c9220d8fef6befa4b","observation_id":"be54c590-61bb-49ff-b72c-860402bd6037","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.13662","last_updated":"2022-02-28T02:03:28Z","snapshot_observed_at":"2026-08-16T19:02:59.631285Z","submitted_at":"2020-11-27T10:57:18Z","title":"FFCI: A Framework for Interpretable Automatic Evaluation of Summarization","version":3},"cited_work":{"arxiv_id":"2011.13662","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2011.13662","snapshot_observed_at":"2026-07-02T20:37:22.672378Z","title":"arXiv preprint arXiv:2011.13662 , year=","venue":null,"work_id":"8393a32b-c97c-45ff-b248-1710ac461d58","year":2011},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":97,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"cited_paper":"/paper/2011.13662","citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:1d20ddf8f03a49d4175fbc4709f7a2c361fe3c56b9fad9c20c9665676573cb82","observation_id":"cca0ce1c-c752-4e57-a8d3-76d7e35c5565","resolution":{"observed_at":"2026-07-02T20:37:22.673684Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"Journal of the Royal Society of Medicine , volume =","venue":null,"work_id":null,"year":2003},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":98,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:05730606496f4a53b75888dc83d0ba9fd4a51fdb644b5414be5b763fda6fa7f5","observation_id":"e2964047-2545-45f2-aa8e-40f212b2f934","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2019.100443","doi":"10.1016/j.conctc.2019.100443","metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"The significant cost of systematic reviews and meta-analyses: A call for greater involvement of machine learning to assess the promise of clinical trials , journal =","venue":"Contemporary Clinical Trials Communications","work_id":"b03c44b9-3ecc-4a54-87ac-66c7245eefaa","year":2019},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":99,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:3f6deba00d2fc4d9ddbc31af6e6e1c19623cfaf941cb6373b063a74d43da4aeb","observation_id":"556ded4f-87cc-4e5c-97f3-cf358ea1493c","resolution":{"observed_at":"2026-06-27T20:21:13.055005Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[{"edge_observation":{"observed_at":"2026-07-11T11:51:01.37004+00:00","source":"paper_reference_links","state":"open"},"event_date":"2019-09-12","event_type":"correction","notice_doi":"10.1016/j.conctc.2019.100450","provenance":{"observed_at":"2026-07-11T03:16:42.407043+00:00","source":"crossref","source_record_id":"10.1016/j.conctc.2019.100450->10.1016/j.conctc.2019.100443:correction"}}],"reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-06-27T20:20:08.996005Z","title":"npj Digital Medicine , volume=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","version":2},"reference_index":100,"source":"arxiv_source","source_observed_at":"2026-06-27T20:20:08.996005Z"},"links":{"citing_paper":"/paper/2606.07936"},"observation_digest":"sha256:dc277f58c1e686143a23c8052b514f291388781bd7fb2ba02ac8aae865433220","observation_id":"7a04601b-ddd8-4b7d-9cd9-903a4cb5e19e","resolution":{"observed_at":"2026-06-27T20:20:08.996005Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2606.07936","last_updated":"2026-06-09T12:36:24Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T22:03:51.528110Z","submitted_at":"2026-06-06T01:55:56Z","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":11,"parse_uncertain":0,"unresolved":63,"verified_exact":26,"verified_fuzzy":0},"total_outbound_references":140},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"thesis":"As of 24 August 2026, this Paper Citation Record lists 100 of 140 outbound references and 0 inbound Pith citation observations for arXiv:2606.07936."}