{"as_of":"2026-08-13T10:11:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:443ce23c71622d664be262c4a34cdf7b5abe529afc7f51343bb002d2d59aa8c4","coverage":[{"denominator":147,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T14:14:09.214957Z","state":"measured"},{"denominator":100,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":100,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.15634/citation-record","integrity":"/paper/2411.15634/integrity","json":"/paper/2411.15634/citation-record.json","paper":"/paper/2411.15634"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.932253Z","title":"online\" 'onlinestring :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.932253Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:acce5ef0315d08851c6007f8474b2b3ab21389e9a30267002f12173eb3bd7fa9","observation_id":"d3bbedf6-79da-44f1-a073-c8fb80e948f5","resolution":{"observed_at":"2026-08-12T14:14:08.932253Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.935276Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.935276Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:49f7602cb596d9980f2d3934cf8faddbb3d4b404119e2a7ed106e37758508524","observation_id":"0b60fde8-63b8-4a72-ad12-297dd0acf522","resolution":{"observed_at":"2026-08-12T14:14:08.935276Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.48550/arxiv.2301.10684","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:18:38.108384Z","title":null,"venue":null,"work_id":"9e862592-e6b8-409f-afd7-57e957a14267","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.937825Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:db4676a63820693529528e4cf2e1a1963e1dee2c8e6ebf09dc78af94927a3292","observation_id":"2d27847f-0aeb-4867-9b4c-2a60be69eb8f","resolution":{"observed_at":"2026-08-12T14:18:38.189288Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.940582Z","title":"Adams, Mark Wilson, and Wen-chung Wang","venue":null,"work_id":null,"year":1997},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.940582Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7c6079f13690dadd69d985bab125e48610e772bdf7cb9525d78a624aff3f1a4c","observation_id":"e96e75a5-d49d-46dd-b075-e2996a270651","resolution":{"observed_at":"2026-08-12T14:14:08.940582Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.03292","last_updated":"2020-11-06T13:40:14Z","snapshot_observed_at":"2026-07-06T07:06:35.439966Z","submitted_at":"2018-10-08T07:27:11Z","title":"Sanity Checks for Saliency Maps","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.03292","snapshot_observed_at":"2026-08-12T14:14:08.943711Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.943711Z"},"links":{"cited_paper":"/paper/1810.03292","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d539c0d427e24450da31c88a3926fcee7fdde54bce80182663ba9a98f2cb0db6","observation_id":"53846e1d-5893-4a9a-a9b3-f3639e2e5575","resolution":{"observed_at":"2026-08-12T14:14:08.943711Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.947350Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.947350Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:253e695f102cac66992b6393e2d2c0366f4baa45d4279a3a0215b9bc4e97e63f","observation_id":"f8d5b14c-2b90-4df9-a3b1-0c3ded31e6c5","resolution":{"observed_at":"2026-08-12T14:14:08.947350Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.950892Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.950892Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5d9c973328f3c929668680c945ac8767183705a7acb160e6f36bd1a84e6f5a1b","observation_id":"1a2d0a2b-db78-4691-86c8-ab033f8a3697","resolution":{"observed_at":"2026-08-12T14:14:08.950892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.954122Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.954122Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b09d9d899f2341731b0db4ceb4c9458073fa0e248ea05db510b4c50f914ae4a0","observation_id":"329d69f7-9eb5-40b8-8364-ae8efd56136c","resolution":{"observed_at":"2026-08-12T14:14:08.954122Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.957274Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.957274Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:0a781f65c00b5c8773e0471693ee1e89cd30910a9633c5bd760200d73b290eb1","observation_id":"e01229e8-3aba-469b-b27b-12c6946860ea","resolution":{"observed_at":"2026-08-12T14:14:08.957274Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.16133","last_updated":"2022-11-30T09:15:10Z","snapshot_observed_at":"2026-08-12T23:53:19.450185Z","submitted_at":"2022-10-28T14:01:32Z","title":"Stop Measuring Calibration When Humans Disagree","version":2},"cited_work":{"arxiv_id":"2210.16133","doi":"10.48550/arxiv.2210.16133","metadata_source":"pith","pith_arxiv_id":"2210.16133","snapshot_observed_at":"2026-08-12T18:16:22.223912Z","title":"Stop Measuring Calibration When Humans Disagree","venue":"cs.CL","work_id":"8e16fe24-5664-484b-b081-ffd9aa2d7002","year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.959989Z"},"links":{"cited_paper":"/paper/2210.16133","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:045cbd40b68c911e9bbe0cecc1d2770069e4144ea2fd983f4361fa619f37357f","observation_id":"bb432597-0eef-421d-ad93-ebfc8bf5e1f9","resolution":{"observed_at":"2026-08-12T14:18:37.880314Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.16102","last_updated":"2024-02-25T15:00:13Z","snapshot_observed_at":"2026-08-13T05:53:56.705175Z","submitted_at":"2024-02-25T15:00:13Z","title":"Interpreting Predictive Probabilities: Model Confidence or Human Label Variation?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.16102","snapshot_observed_at":"2026-08-12T14:14:08.963443Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.963443Z"},"links":{"cited_paper":"/paper/2402.16102","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a38ec84e16899cd916da301cec23cc8aeff43aafe3c9dd5d9f424b903e6c70ec","observation_id":"764eeefd-4ed4-4f10-b03e-bbedfd8cdc45","resolution":{"observed_at":"2026-08-12T14:14:08.963443Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3386/w23478","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.878857Z","title":"Chin, Thomas J","venue":null,"work_id":"de926193-1b54-441c-818f-8f9be5599938","year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.966386Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e23500d8cefc4e532e19e57e40e3fb6b27548f0e085df92cb87c45ce74b6214d","observation_id":"06c0eb78-89be-4192-8748-e3b5d803c54a","resolution":{"observed_at":"2026-08-12T14:14:09.881615Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.969771Z","title":"Chin, Thomas J","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.969771Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7c71c7dbed0ef22836b38d754f63d08a9061a7e5930d784dee2c961836a5e049","observation_id":"544bf104-fad7-4152-b825-8c2ef0b84b9f","resolution":{"observed_at":"2026-08-12T14:14:08.969771Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.973174Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.973174Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:15b6a5e7eade1bf05b713c5f3e9e0c1bb8b3bfd1b0f4c2261f4dbc1c0b165cdf","observation_id":"0ea02359-f3e4-4344-b869-4a5d0e311b43","resolution":{"observed_at":"2026-08-12T14:14:08.973174Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.975460Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.975460Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ee4fcdba277009dac89d2f1630f55b8a9d9cb661ac34a1ea4bd593d071b9fc3f","observation_id":"9fe81829-d64e-483d-8c6e-54a806e7bd72","resolution":{"observed_at":"2026-08-12T14:14:08.975460Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.977909Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.977909Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c232e5bafb7fb5e3cf8b587f18873eda2500dbd660dc0a42ef33805a5a872f1c","observation_id":"8aba37f7-ac55-4b65-8be1-a8e93c6fad9f","resolution":{"observed_at":"2026-08-12T14:14:08.977909Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.981168Z","title":"Williamson, and and Robert J","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.981168Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:883b68862e008549f7f435b4bb23fb17a4ef195e2d0b3d1aedbaa7b47d8f0653","observation_id":"95e7f13e-cccc-4ce5-930f-b873d4a24225","resolution":{"observed_at":"2026-08-12T14:14:08.981168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.983702Z","title":"Howcroft","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.983702Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:779b8ae5b15a8c7992ef1f826eb82c123d67954c48b088a2bd2c33089f69a8cf","observation_id":"6090a068-df32-4fbd-a367-ae4f25d035a5","resolution":{"observed_at":"2026-08-12T14:14:08.983702Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.986305Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.986305Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7d3569578a4b1d542e6ea2687b741dfca926ba34f4ca111ea22e70aff9145827","observation_id":"f31e83c6-1554-4910-93fe-bdd1a9fbc11d","resolution":{"observed_at":"2026-08-12T14:14:08.986305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.20268","last_updated":"2025-04-28T11:50:42Z","snapshot_observed_at":"2026-08-12T22:14:34.277092Z","submitted_at":"2024-10-26T20:39:41Z","title":"Centaur: a foundation model of human cognition","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.20268","snapshot_observed_at":"2026-08-12T14:14:08.992453Z","title":"Eckstein, Noémi Éltető, Thomas L","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.992453Z"},"links":{"cited_paper":"/paper/2410.20268","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d43e444f0d288534f9a5c42a87d7d9650aa0e2dd459393175a890a8088d44d13","observation_id":"2f2a0c7f-99c8-49f3-834f-f067e2594183","resolution":{"observed_at":"2026-08-12T14:14:08.992453Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.15590","last_updated":"2022-06-21T17:48:26Z","snapshot_observed_at":"2026-08-12T12:44:22.330348Z","submitted_at":"2021-06-29T17:24:14Z","title":"The Values Encoded in Machine Learning Research","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.15590","snapshot_observed_at":"2026-08-12T14:14:08.995399Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.995399Z"},"links":{"cited_paper":"/paper/2106.15590","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:139685e9362a8c7127d103087919326e0265aad3597591a89e88a9e69efd5a36","observation_id":"0e5030af-af46-4865-81d7-2c24413bdc18","resolution":{"observed_at":"2026-08-12T14:14:08.995399Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1162/edfp_a_00251","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.848827Z","title":null,"venue":null,"work_id":"de2a692d-4b97-4c3d-9f16-f30488b54112","year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.997769Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:afd9fc525874621f65190b7676aa726ec67594beb2445849770026853395d80a","observation_id":"11072218-a1bd-4d8d-8c97-9e54122a8a3e","resolution":{"observed_at":"2026-08-12T14:14:09.851093Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.999694Z","title":"Charalambous, and Heather C","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.999694Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e37f36ec85283abeaa5e93c024585c437dd05fde02f4325bc0e0d6f0039096fe","observation_id":"ac9939bb-bc79-4b56-89f7-b9e7e78073b1","resolution":{"observed_at":"2026-08-12T14:14:08.999694Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.001996Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.001996Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:10b16f205099b0198bfa4d442344ddc4a523ba878c7c656a87f889bb72a385ee","observation_id":"8ccb3262-08df-45ad-994f-05809d624e0c","resolution":{"observed_at":"2026-08-12T14:14:09.001996Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.003948Z","title":null,"venue":null,"work_id":null,"year":2001},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.003948Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4031d90e4155109de68e3718ffa8915008144e596953fe9f556d0b41fd4c198f","observation_id":"0b6d44ed-e166-4446-8cc5-3c31dd5291a1","resolution":{"observed_at":"2026-08-12T14:14:09.003948Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/978-1-4757-3456-0_6","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.837248Z","title":null,"venue":null,"work_id":"e8b1fc36-81d1-4362-a2e3-b95953013841","year":2001},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.006003Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4d8bd6761c794a24521bc0d36c085aad8ee1087db72bdd1876db29283905462f","observation_id":"eeafe266-df5f-4a82-9800-ff37a02c99eb","resolution":{"observed_at":"2026-08-12T14:14:09.840105Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.008003Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.008003Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:fe3458e80bf40075cdc063fe43c5c9d38df21d6f7721ca11c3174b172df82972","observation_id":"daecf482-9a0e-4fa4-9cce-f89fb007089c","resolution":{"observed_at":"2026-08-12T14:14:09.008003Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.010344Z","title":"Briggs and Mark Wilson","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.010344Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d176fba53ddaf07e6ab6435607ffc386f27a73843f2779a00504ffedaade2952","observation_id":"35fa43e3-5d0d-490f-8cf5-e92631b2b074","resolution":{"observed_at":"2026-08-12T14:14:09.010344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1111/emip.12478","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.828774Z","title":"Casabianca","venue":null,"work_id":"6c317141-df30-4386-9968-c497cc59c3ba","year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.013050Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7bc6f99589de5c3082f04d63d43fa0f1b48a678a43645396700eea8d88c65f30","observation_id":"af6b96f7-1b39-4ff2-8a68-0ac167d1df9f","resolution":{"observed_at":"2026-08-12T14:14:09.831925Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1177/0013164413486987","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.820594Z","title":"Casabianca, Daniel F","venue":null,"work_id":"5d337260-3686-41d8-b297-c6f803706488","year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.015935Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dd8d110e2e74a44d1ca58f6265bb0ca2a93f14ca0ffc5bd72e6216855245009d","observation_id":"e611b204-bae3-4555-8638-3ed51b32f23b","resolution":{"observed_at":"2026-08-12T14:14:09.823640Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1163/9789004418875_014","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.811415Z","title":"Charalambous and Seán Delaney","venue":null,"work_id":"31388d5b-e8f9-4d78-9318-aa3e2adfe250","year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.019427Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9b49f09d5670094460ef4205ae8807bd56dcfea68d486c55bd9d92f79282f3ea","observation_id":"0b500ddc-90e0-437c-87a7-fbd377b50cbe","resolution":{"observed_at":"2026-08-12T14:14:09.814548Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1037/1082-989x.10.2.206","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.802711Z","title":null,"venue":null,"work_id":"7d09242a-db68-4d2c-9c2f-24a37b3664bb","year":2005},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.022707Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:2364a970e599070aa7ecccb2922ea44855398e091db93fede3d7249d0ea05ef1","observation_id":"0c269d01-8519-4737-92ff-bd507f0a83ae","resolution":{"observed_at":"2026-08-12T14:14:09.806031Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1808.00023","last_updated":"2023-08-14T19:14:00Z","snapshot_observed_at":"2026-07-06T06:53:21.002757Z","submitted_at":"2018-07-31T18:38:04Z","title":"The Measure and Mismeasure of Fairness","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1808.00023","snapshot_observed_at":"2026-08-12T14:14:09.025189Z","title":"Gaebler, Hamed Nilforoshan, Ravi Shroff, and Sharad Goel","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.025189Z"},"links":{"cited_paper":"/paper/1808.00023","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:36ae157b64e2ba2a11e323ecd92bb8c5387aa807be70586285be242d5cb9e53b","observation_id":"44f0441b-d2b5-42aa-94a5-ca6bf6459fbe","resolution":{"observed_at":"2026-08-12T14:14:09.025189Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s11336-024-09955-8","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.788085Z","title":null,"venue":null,"work_id":"7dde3b8f-200f-4b76-a6d8-badf7cd57103","year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.028229Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6dc8c8c318ba487adc27720820301d7cbfad316027b2bdf72cd9d75d842143b9","observation_id":"50154bf5-db45-4aeb-b8e4-0a2f745a53e2","resolution":{"observed_at":"2026-08-12T14:14:09.790912Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2011.03395","last_updated":"2020-11-24T19:16:02Z","snapshot_observed_at":"2026-08-07T12:11:27.017871Z","submitted_at":"2020-11-06T14:53:13Z","title":"Underspecification Presents Challenges for Credibility in Modern Machine Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.03395","snapshot_observed_at":"2026-08-12T14:14:09.030940Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.030940Z"},"links":{"cited_paper":"/paper/2011.03395","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:135f3f362c92648cf28ca6bc95c8c8fa407ce314e37967c5153ab3145f1ec5f4","observation_id":"3ef5eb04-d820-4ac1-a0c4-69a7d3b40857","resolution":{"observed_at":"2026-08-12T14:14:09.030940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.033901Z","title":null,"venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.033901Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:919c40f107ef689111b5bb37f4cb08c829b3f9c4ec8f56a131dffd9ae0a09b73","observation_id":"d3474244-fa02-47cf-aa99-4f3421691efa","resolution":{"observed_at":"2026-08-12T14:14:09.033901Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.036476Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.036476Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ffa18492f9aaed987e0e347f1315ceb1720dbff071afe7a3b5335cbf2a464b8f","observation_id":"675e123f-daaa-46c0-97d6-75ee9988bfd6","resolution":{"observed_at":"2026-08-12T14:14:09.036476Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3758/bf03195496","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.773933Z","title":null,"venue":null,"work_id":"eb7d17a3-3773-4d28-ac9d-6fb2d2e0b7bd","year":2003},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.039424Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b25be53345507d0a592ff2095f77d4230a576cc51a80fa0c979d3056f5226434","observation_id":"a76ef933-6240-48b3-aac6-b2e82fb86807","resolution":{"observed_at":"2026-08-12T14:14:09.776086Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.042096Z","title":null,"venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.042096Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:792cb96e81b1f8f961f6bb2517423f2b1918c77c51d29eb580b9a624ffd063bd","observation_id":"0dbac9a9-8bd6-4c40-a56a-94c40a7ddccf","resolution":{"observed_at":"2026-08-12T14:14:09.042096Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1111/jedm.12358","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.767304Z","title":null,"venue":null,"work_id":"a57b84ee-488f-4431-afd4-65cf3f04972b","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.044529Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6cc5fe8564c4678d8b5230b7dde9551e178eb9188f9a3efe6fe9f8a3537f82a5","observation_id":"85e89cf4-d389-4670-b3f7-3787d03aa71d","resolution":{"observed_at":"2026-08-12T14:14:09.769712Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.047815Z","title":"DeCarlo, YoungKoung Kim, and Matthew S","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.047815Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f23c3ad8397f6cafa15c7bbf57abff9786aed8a2a9abee5762c0bedd485c9cee","observation_id":"18aa29b8-4c7d-4e2c-b8c1-9e6bdff76ac7","resolution":{"observed_at":"2026-08-12T14:14:09.047815Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.11772","last_updated":"2023-05-24T18:41:18Z","snapshot_observed_at":"2026-07-06T14:21:22.601555Z","submitted_at":"2022-11-21T19:00:01Z","title":"The NCTE Transcripts: A Dataset of Elementary Math Classroom Transcripts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.11772","snapshot_observed_at":"2026-08-12T14:14:09.050988Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.050988Z"},"links":{"cited_paper":"/paper/2211.11772","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:24fd5e31cffe78e3abdd1f859f7993bb962ca12020ea58fbb85bf2849fee3fa6","observation_id":"1a8a9b2c-826e-4a08-ad5b-82ac0974e7a1","resolution":{"observed_at":"2026-08-12T14:14:09.050988Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.053817Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.053817Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dbf8a318a1f334b1bf86448f4b8e48c6656fde932637ea903c5c36d110e5974c","observation_id":"043c85a1-f15f-4c7e-b04a-0df0dc8c4bd6","resolution":{"observed_at":"2026-08-12T14:14:09.053817Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.060219Z","title":"Hill, Shyamoli Sanghi, and Ariel Chung","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.060219Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:135e3d7aff9f3c3ade89f238b96736986d6378ac88a9f797157573d4af45a8b9","observation_id":"cdadb409-5722-431f-8152-f1b6b2736df6","resolution":{"observed_at":"2026-08-12T14:14:09.060219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.03873","last_updated":"2021-06-07T18:00:06Z","snapshot_observed_at":"2026-08-12T18:35:17.230557Z","submitted_at":"2021-06-07T18:00:06Z","title":"Measuring Conversational Uptake: A Case Study on Student-Teacher Interactions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.03873","snapshot_observed_at":"2026-08-12T14:14:09.062876Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.062876Z"},"links":{"cited_paper":"/paper/2106.03873","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:68d3aeae675954f32d0d8e6e8858f4801bdc8553c5a687bf31ca61fed7ab2e19","observation_id":"174fb693-b55e-4da7-810f-f7010902b607","resolution":{"observed_at":"2026-08-12T14:14:09.062876Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.065992Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.065992Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5dec933647111b8bf075d1118d6b7dd15c618d5c14b7b16212d9b16b26ca1a75","observation_id":"79ab10e0-4033-4452-9be1-11ef46fb6c6c","resolution":{"observed_at":"2026-08-12T14:14:09.065992Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.068173Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.068173Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c089bd64f466fa9832efa0a9e0a344c744e1eede3b501c795fa0edea20928969","observation_id":"e6ae7ae8-f87c-411a-9b37-803d6566e0ab","resolution":{"observed_at":"2026-08-12T14:14:09.068173Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.04884","last_updated":"2022-01-09T20:58:09Z","snapshot_observed_at":"2026-08-10T18:05:22.148964Z","submitted_at":"2021-08-10T19:19:41Z","title":"Retiring Adult: New Datasets for Fair Machine Learning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.04884","snapshot_observed_at":"2026-08-12T14:14:09.070435Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.070435Z"},"links":{"cited_paper":"/paper/2108.04884","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e8e282bcd58c91b3553aef6cebd642f59f14772a7dc419934d6e5d891be6dc58","observation_id":"b1731622-ac80-4274-a663-ba213a473dc6","resolution":{"observed_at":"2026-08-12T14:14:09.070435Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.073853Z","title":"Donnelly, Nathaniel Blanchard, Andrew M","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.073853Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:333a32900e2a552a6cee5e2f190f003101e0bd2d6a39ae05737b74156320fe93","observation_id":"6d2aae0d-b564-4a02-9410-a3000608a9bc","resolution":{"observed_at":"2026-08-12T14:14:09.073853Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.076972Z","title":null,"venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.076972Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7869545342437e8e5cd6475a936614aec93f0b5d43984377b2a3a892de5ea253","observation_id":"ad7d84f9-f4bf-4777-8f70-fa5d852cffcf","resolution":{"observed_at":"2026-08-12T14:14:09.076972Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.080063Z","title":"Detecting Illusory Halo Effects in Rater - Mediated Assessment : A Mixture Rasch Facets Modeling Approach","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.080063Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7fe34fc6d142ea44958d7507a1e2bb7a190548c0ad6085bff2aa5bbf28ca8f7a","observation_id":"b1b1208f-6afd-4e8e-a2db-05d3baa6a897","resolution":{"observed_at":"2026-08-12T14:14:09.080063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.083613Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.083613Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:42052bb44898f0d9ddf4e25a7c435a8761ac43d556f58b66fec5c39a07bfdd7d","observation_id":"36885db8-7e67-4d6f-88de-de35f38a7b54","resolution":{"observed_at":"2026-08-12T14:14:09.083613Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08818","last_updated":"2024-09-17T05:29:50Z","snapshot_observed_at":"2026-08-12T23:43:56.489990Z","submitted_at":"2024-06-13T05:20:42Z","title":"Linguistic Bias in ChatGPT: Language Models Reinforce Dialect Discrimination","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08818","snapshot_observed_at":"2026-08-12T14:14:09.086235Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.086235Z"},"links":{"cited_paper":"/paper/2406.08818","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:fc0f3586c0a884d08249459055d047cd4dc077ccde36e20e5ace8100ddd8e878","observation_id":"f14b809e-c568-4ddc-8322-15081224ee81","resolution":{"observed_at":"2026-08-12T14:14:09.086235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.089727Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.089727Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6d82f9b3d6f750e62bf12230ea81047afef66e17cc141d160b91663fb99a3edf","observation_id":"7267ecc7-a057-4940-abe4-19e3b4c4c7c4","resolution":{"observed_at":"2026-08-12T14:14:09.089727Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.14375","last_updated":"2022-09-28T19:04:43Z","snapshot_observed_at":"2026-08-11T23:28:54.309888Z","submitted_at":"2022-09-28T19:04:43Z","title":"Improving alignment of dialogue agents via targeted human judgements","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.14375","snapshot_observed_at":"2026-08-12T14:14:09.092248Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.092248Z"},"links":{"cited_paper":"/paper/2209.14375","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:04c622af3a649bfc18021d9b57cfe8aee472ca444e9012b13c7de9f3d00abb86","observation_id":"b5bc1457-cec8-4ca7-9631-d3b204052af1","resolution":{"observed_at":"2026-08-12T14:14:09.092248Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.095272Z","title":"Gordon, Michelle S","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.095272Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9fddaf1b5114ebb968c4775a9b60e114c7d5c2dddefcdc05e51e63bad1df4bc9","observation_id":"cd259315-3192-47fc-932e-d4ee47f352f6","resolution":{"observed_at":"2026-08-12T14:14:09.095272Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.097777Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.097777Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:47080c720ecddbecddf12b1093819094e42c66ca13568a0b0c187416914c346c","observation_id":"e88e1b8b-318b-4843-8643-967c012e8e04","resolution":{"observed_at":"2026-08-12T14:14:09.097777Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/bf02288892","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.717199Z","title":null,"venue":null,"work_id":"96df39f9-fed8-410d-b8d6-654fa976f943","year":1945},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.100749Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6b6086ff93b45a65b8c8aa0da75235e64bd70ebcdb507b62b46f6df8e2255f57","observation_id":"8be0001f-ad92-42a4-90e3-ff775b6a7cbf","resolution":{"observed_at":"2026-08-12T14:14:09.720638Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.104380Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.104380Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:cba9a4594b6e6585e279191fd4e7252231997a8c3b1c315cc77cae086e2a59fc","observation_id":"6cbb5cb8-a405-4ef9-a34e-9f69ae17a301","resolution":{"observed_at":"2026-08-12T14:14:09.104380Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1610.02413","last_updated":"2016-10-07T20:16:29Z","snapshot_observed_at":"2026-07-06T05:13:49.420787Z","submitted_at":"2016-10-07T20:16:29Z","title":"Equality of Opportunity in Supervised Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1610.02413","snapshot_observed_at":"2026-08-12T14:14:09.107756Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.107756Z"},"links":{"cited_paper":"/paper/1610.02413","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dacee85af708929e82be6eec92cefbdcd3f42f26f9fdf3b2ddb50b7749aa6bbe","observation_id":"d64aecd0-3008-4729-a79a-95ee69bcd88e","resolution":{"observed_at":"2026-08-12T14:14:09.107756Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2112.11973","last_updated":"2021-12-22T15:44:30Z","snapshot_observed_at":"2026-08-12T01:51:09.794653Z","submitted_at":"2021-12-22T15:44:30Z","title":"Toward Educator-focused Automated Scoring Systems for Reading and Writing","version":1},"cited_work":{"arxiv_id":"2112.11973","doi":"10.48550/arxiv.2112.11973","metadata_source":"pith","pith_arxiv_id":"2112.11973","snapshot_observed_at":"2026-08-12T18:16:22.223912Z","title":"Toward Educator-focused Automated Scoring Systems for Reading and Writing","venue":"cs.CL","work_id":"40e1d755-80cf-4160-bbb7-e4284ddf7522","year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.110605Z"},"links":{"cited_paper":"/paper/2112.11973","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:95d24e87ed737e6afd6dbc2879c52ff2f397a5434b0ce1e29ea57bdbfffc291c","observation_id":"dc0706ab-729d-4d53-8209-76c186711ea2","resolution":{"observed_at":"2026-08-12T14:14:09.705374Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.112901Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.112901Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:706748bb95392036733ec6538a74af9fd690ba11aca79926ad28308ec59aa0c3","observation_id":"f94444af-1cd2-4355-93f0-11241e196ab1","resolution":{"observed_at":"2026-08-12T14:14:09.112901Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.114895Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.114895Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b1ea02a0edc4cdd018699fc0bff11c8db45ecaa113564d2614bf3ce9a2b1d8b8","observation_id":"71f71d33-b8b1-4568-b4cd-17fb1fd6766b","resolution":{"observed_at":"2026-08-12T14:14:09.114895Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.14516","last_updated":"2025-03-28T15:40:49Z","snapshot_observed_at":"2026-08-12T22:20:02.504110Z","submitted_at":"2024-10-18T14:55:14Z","title":"Do LLMs \"know\" internally when they follow instructions?","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.14516","snapshot_observed_at":"2026-08-12T14:14:09.116865Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.116865Z"},"links":{"cited_paper":"/paper/2410.14516","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6400e1d33eb9448659fa738f5bc01b99b905e3fe060b7c0ff5f7fb48ac7dfd09","observation_id":"19d33387-39af-4cdf-98fd-a09c0e9d5a32","resolution":{"observed_at":"2026-08-12T14:14:09.116865Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.119103Z","title":"Hill, Merrie L","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.119103Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3df992f69fefceb57327b8fccf9afa3c658b48186da0c9d2f240c1961234997c","observation_id":"17f6823b-b6f7-41f9-b09e-f106d60d0024","resolution":{"observed_at":"2026-08-12T14:14:09.119103Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.121664Z","title":"Hill, Charalambos Y","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.121664Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:da7c8649dbb428268cfaaef0e2f97566eebf3a61f5ec28e055f424846b4e38bc","observation_id":"e479c0b1-fa07-4a46-bfb0-5697962bacbb","resolution":{"observed_at":"2026-08-12T14:14:09.121664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3102/0013189x12437203","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.690695Z","title":"Hill, Charalambos Y","venue":null,"work_id":"a610c88f-0ed2-43c6-b149-5c90027a762f","year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.125222Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dacad269e4af987c6aed3c3694b29a33e795153067dff76b637a803036abcf73","observation_id":"a634179d-2636-452b-a7b5-9b5af09a1ec0","resolution":{"observed_at":"2026-08-12T14:14:09.693991Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.129009Z","title":"Ho and Thomas J","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.129009Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c8b0012184212be3965e0f75e0f3f8dcd9d390d97ef570850fb0ee5b6996c305","observation_id":"89f67e84-f0ed-4f6b-94dd-02229a9e3d7c","resolution":{"observed_at":"2026-08-12T14:14:09.129009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.131853Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.131853Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b3e4bea42e1115ff77debf78f963d9abdbae8309d1593d13be049a4f190a2f85","observation_id":"4b9dc2b6-a12f-4325-bf96-3ec909ed363a","resolution":{"observed_at":"2026-08-12T14:14:09.131853Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.00742","last_updated":"2024-03-01T18:43:09Z","snapshot_observed_at":"2026-08-13T04:05:21.054158Z","submitted_at":"2024-03-01T18:43:09Z","title":"Dialect prejudice predicts AI decisions about people's character, employability, and criminality","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.00742","snapshot_observed_at":"2026-08-12T14:14:09.134401Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.134401Z"},"links":{"cited_paper":"/paper/2403.00742","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4ec4b1aedd8cbd99081f362fdd5d35ae30db0aef186cdd227cf75336ff80db51","observation_id":"64172633-dfc9-4e12-a695-d300d9c3410d","resolution":{"observed_at":"2026-08-12T14:14:09.134401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16349","last_updated":"2024-01-16T16:51:33Z","snapshot_observed_at":"2026-08-13T10:03:12.912726Z","submitted_at":"2023-09-28T11:18:20Z","title":"Human Feedback is not Gold Standard","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16349","snapshot_observed_at":"2026-08-12T14:14:09.138144Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.138144Z"},"links":{"cited_paper":"/paper/2309.16349","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:487a7af9f1c963ebab04fe01fe9d7700474f0e7aec849eda99b438ae91e4e8c6","observation_id":"745caea2-c5bb-4f3f-9dfc-c3816e4c9293","resolution":{"observed_at":"2026-08-12T14:14:09.138144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.140999Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.140999Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:eaa31b422119cae387bd6ad538b0c70aa84538222704c84ab330bd5e045fa18f","observation_id":"c2598ee1-4bf0-4549-8c1e-d9b7d0475061","resolution":{"observed_at":"2026-08-12T14:14:09.140999Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3758/s13428-023-02275-2","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.668456Z","title":null,"venue":null,"work_id":"51c9e345-6be0-4055-a7fe-c4e82bb5770b","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.143596Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:73cd6a36be6a191ba1b951b9d3350ca9f3098f85eb35629a8c04e0bd0fb0e57d","observation_id":"9227160f-b29e-441c-be05-b3af01b69261","resolution":{"observed_at":"2026-08-12T14:14:09.670960Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.146120Z","title":"Jacobs, Ryan J","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.146120Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:78f9ba0102cb6a602afc72e1d9e3e408d3c2822a3217acf92201fa1e6678a918","observation_id":"8b4f7385-7b87-4781-87f5-2e4352df86c3","resolution":{"observed_at":"2026-08-12T14:14:09.146120Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.14288/1.0437518","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.659135Z","title":null,"venue":null,"work_id":"b77bb800-1c77-42f4-b08d-5c1699ff1ef8","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.148663Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:336fd9ec7627a3df29d0758451130b76027a829e8b029c1ea655284db6ec716a","observation_id":"ff19faca-0d5e-4022-ab32-cddd4fc7742f","resolution":{"observed_at":"2026-08-12T14:14:09.663305Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.151878Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.151878Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5501a1eaea0779f4359e5920642a1f918ab1e5550409c688ff5fbe45ef70f999","observation_id":"84eac20c-2ad6-4870-9895-5ff169d9204e","resolution":{"observed_at":"2026-08-12T14:14:09.151878Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3886/icpsr36095.v4","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.650979Z","title":null,"venue":null,"work_id":"f8436c10-3499-485a-b647-f689c32fd268","year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.154478Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:30c581add904ef56cbfec88bca212b799805812b0e815fad3b7e324314a0e7b3","observation_id":"976f21b2-d776-48b9-a956-4181e1006657","resolution":{"observed_at":"2026-08-12T14:14:09.654223Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.157097Z","title":"Kane, Daniel F","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.157097Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:0d0dcb229f9f2ae1c4819d324ae7e275a81c4233687e5ba61df022991513803a","observation_id":"a1ecae9d-d59b-44e0-a90e-83d6e767a5da","resolution":{"observed_at":"2026-08-12T14:14:09.157097Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.159929Z","title":"Kane and Douglas O","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":81,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.159929Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:809dc24ea1fd151f93b6a6345a99dabd8a632d11f3af302f123913fd815e1ee6","observation_id":"fecb4ad0-2c0b-40a8-b400-d0083e08f04f","resolution":{"observed_at":"2026-08-12T14:14:09.159929Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.161958Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":82,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.161958Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:401c398e6ffac72f6ffa5586b85fad5daa62038c59dfbe700ad8510b9485aee2","observation_id":"86ab413f-a95b-41a5-b04b-913c8eea7065","resolution":{"observed_at":"2026-08-12T14:14:09.161958Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.164025Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":83,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.164025Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:375aa7234e6c99547b2fc3448155973a9831ef0200c4688f26e9526e7ae56034","observation_id":"d73ac0b4-0b64-4045-95cc-19ba4e1b3007","resolution":{"observed_at":"2026-08-12T14:14:09.164025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.166358Z","title":"Olney, Patrick Donnelly, Martin Nystrand, and Sidney K","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.166358Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3df694e15a539b5fbc49e39eaaf3d0b933f69b14bc789f2e29cbd5635db6f9cc","observation_id":"e09eab0a-0f58-434e-8920-5153488f7288","resolution":{"observed_at":"2026-08-12T14:14:09.166358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.14337","last_updated":"2021-04-07T17:49:17Z","snapshot_observed_at":"2026-08-03T23:24:34.241550Z","submitted_at":"2021-04-07T17:49:17Z","title":"Dynabench: Rethinking Benchmarking in NLP","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.14337","snapshot_observed_at":"2026-08-12T14:14:09.168262Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":85,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.168262Z"},"links":{"cited_paper":"/paper/2104.14337","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:accc02f2d0c2ed6a56cd7ca5621a33b740263068f0b92d1952dc7b9bc87110b7","observation_id":"f15163c3-9e9c-41e5-a705-858294c34b73","resolution":{"observed_at":"2026-08-12T14:14:09.168262Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1711.11279","last_updated":"2018-06-07T04:33:27Z","snapshot_observed_at":"2026-08-04T20:10:18.590341Z","submitted_at":"2017-11-30T09:26:12Z","title":"Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV)","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1711.11279","snapshot_observed_at":"2026-08-12T14:14:09.171046Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":86,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.171046Z"},"links":{"cited_paper":"/paper/1711.11279","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9828841e3726b23f7d04e911d0d6b2412deab9cca96435f60a91ee474dfa609a","observation_id":"7e58f77d-5610-4cb9-b29c-100ba27d35c4","resolution":{"observed_at":"2026-08-12T14:14:09.171046Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6980","last_updated":"2017-01-30T01:27:54Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2014-12-22T13:54:29Z","title":"Adam: A Method for Stochastic Optimization","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6980","snapshot_observed_at":"2026-08-12T14:14:09.174083Z","title":"Kingma and Jimmy Ba","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.174083Z"},"links":{"cited_paper":"/paper/1412.6980","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:32ef347e2a0079bb415597fd680b6766796c0ce9088ffe6834905492225814f0","observation_id":"a8e4eb6a-89de-490e-8472-8011304fdcdc","resolution":{"observed_at":"2026-08-12T14:14:09.174083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1073/pnas.1212738110","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.618029Z","title":null,"venue":null,"work_id":"52d87162-d004-49fd-94d7-05685d9dcf1c","year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.177851Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a235fe3bb94bdd9dbdc302009aaf8923e3374b8fc87770b54399ce516c13da60","observation_id":"aa4a5d4f-a25e-41e1-b5ec-df41ec7b7c4d","resolution":{"observed_at":"2026-08-12T14:14:09.620934Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.180739Z","title":"Kromrey, Robert H","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.180739Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d8131baaa843baf0e54679b27c9f1ff20c596e67c1d0c7b13d5d6ceab0e5d5db","observation_id":"4089d83d-a529-4d28-b9d9-4da6d547d20b","resolution":{"observed_at":"2026-08-12T14:14:09.180739Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.183188Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":90,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.183188Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e893ab661f695f5ef82c4fd7c36e87ef39e1935cc62acf56fd389a5f736c6c1a","observation_id":"0cd3a3ac-98c9-4a17-8eef-42a1b0e2d732","resolution":{"observed_at":"2026-08-12T14:14:09.183188Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.186148Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.186148Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:674d53d7b3b9961d93709d423e5e76f47f3998c15ee9ef255bd36b7e0698c471","observation_id":"20bfe55c-f1af-43b3-b239-703504d6c0e7","resolution":{"observed_at":"2026-08-12T14:14:09.186148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.03281","last_updated":"2023-08-07T03:52:59Z","snapshot_observed_at":"2026-08-04T23:10:23.964516Z","submitted_at":"2023-08-07T03:52:59Z","title":"Towards General Text Embeddings with Multi-stage Contrastive Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.03281","snapshot_observed_at":"2026-08-12T14:14:09.188693Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":92,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.188693Z"},"links":{"cited_paper":"/paper/2308.03281","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a74d46d31b8f2932676a08d28e697a0e4d23779ede61aec85ca4a8c1ced41cd0","observation_id":"022f8911-eeff-48de-ab48-506de17adb63","resolution":{"observed_at":"2026-08-12T14:14:09.188693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.191656Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":93,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.191656Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a7361de937e4944f8e2b7709150714b241966e892e643e31819b1907cb1e5352","observation_id":"fd527404-979f-4b42-9b1d-ef1def72db24","resolution":{"observed_at":"2026-08-12T14:14:09.191656Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.194277Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":94,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.194277Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b0fad00ce5bc12003395237c1b658c1df92b9e82156ca8d998f16d5e7b9f8348","observation_id":"fadda3b2-aac2-4ac7-91e6-75df02de3ce7","resolution":{"observed_at":"2026-08-12T14:14:09.194277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.03172","last_updated":"2023-11-20T23:09:34Z","snapshot_observed_at":"2026-07-06T15:51:12.179086Z","submitted_at":"2023-07-06T17:54:11Z","title":"Lost in the Middle: How Language Models Use Long Contexts","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.03172","snapshot_observed_at":"2026-08-12T14:14:09.197109Z","title":"Liu, Kevin Lin, John Hewitt, Ashwin Paranjape, Michele Bevilacqua, Fabio Petroni, and Percy Liang","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":95,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.197109Z"},"links":{"cited_paper":"/paper/2307.03172","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6576c2db13590232ce1970c390ff13987b639dd9653eaa193a1cb579d7918e7a","observation_id":"8eb3a78a-58c6-4112-b410-6f179d23e31b","resolution":{"observed_at":"2026-08-12T14:14:09.197109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.16634","last_updated":"2023-05-23T22:12:16Z","snapshot_observed_at":"2026-08-02T04:02:36.848064Z","submitted_at":"2023-03-29T12:46:54Z","title":"G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.16634","snapshot_observed_at":"2026-08-12T14:14:09.199888Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":96,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.199888Z"},"links":{"cited_paper":"/paper/2303.16634","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c17751bccb579bd21d149bf47ca2412072f7279d816f1b5c80ad0c72eb45e406","observation_id":"5e9719f6-69ed-483f-a19d-ea072adccc45","resolution":{"observed_at":"2026-08-12T14:14:09.199888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1907.11692","last_updated":"2019-07-26T17:48:29Z","snapshot_observed_at":"2026-07-31T22:31:37.910868Z","submitted_at":"2019-07-26T17:48:29Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.11692","snapshot_observed_at":"2026-08-12T14:14:09.203006Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":97,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.203006Z"},"links":{"cited_paper":"/paper/1907.11692","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:2b5e005c5345079c7c8034ee99db3a06d7ffd3325504b048f3fb42e8a7a7c97b","observation_id":"14abb02f-0a96-43d5-af03-413d47d90daa","resolution":{"observed_at":"2026-08-12T14:14:09.203006Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1705.07874","last_updated":"2017-11-25T03:53:32Z","snapshot_observed_at":"2026-07-06T05:43:42.722222Z","submitted_at":"2017-05-22T17:38:10Z","title":"A Unified Approach to Interpreting Model Predictions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1705.07874","snapshot_observed_at":"2026-08-12T14:14:09.206025Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":98,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.206025Z"},"links":{"cited_paper":"/paper/1705.07874","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6826b870f51958f09e3c9650b53222031a3498dacc1729ecc80cabbd101b369a","observation_id":"1bf838de-3e95-4d69-84b1-583a98dc8054","resolution":{"observed_at":"2026-08-12T14:14:09.206025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.208438Z","title":"French, and Helen Patrick","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":99,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.208438Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:43e218652e6a54e1c63b2f86c2311d701fb0c2066ff0ba9ce949c9b1d09e20e6","observation_id":"fcd74786-a18c-4bce-8a27-429309a63ae3","resolution":{"observed_at":"2026-08-12T14:14:09.208438Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3102/1076998606298033","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.577671Z","title":"Mariano and Brian W","venue":null,"work_id":"38116f54-7401-4817-99a3-80877d3e8c5c","year":2007},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":100,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.210718Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dd5d1d49f6dcc222988250ad504696b70ec6651ccddc9a65ac4d1c9027ff7d99","observation_id":"f81256ce-2047-45b2-add8-e77e867f9d28","resolution":{"observed_at":"2026-08-12T14:14:09.580641Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.13638","last_updated":"2023-09-24T13:35:28Z","snapshot_observed_at":"2026-08-13T10:06:45.687474Z","submitted_at":"2023-09-24T13:35:28Z","title":"Embers of Autoregression: Understanding Large Language Models Through the Problem They are Trained to Solve","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.13638","snapshot_observed_at":"2026-08-12T14:14:09.212738Z","title":"Thomas McCoy, Shunyu Yao, Dan Friedman, Matthew Hardy, and Thomas L","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":101,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.212738Z"},"links":{"cited_paper":"/paper/2309.13638","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:dc46b5daa42c22bb2102421de00ba00bf28b948a2e7b31a9c44b64c61132f6f1","observation_id":"e1758391-6eef-4955-9203-fb5798cc9dd4","resolution":{"observed_at":"2026-08-12T14:14:09.212738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.214957Z","title":null,"venue":null,"work_id":null,"year":1998},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":102,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.214957Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:650d891342381c373df1279872b8fd409ff153d588bb8697bd2dffa2b4bf09dc","observation_id":"224c9407-044d-446b-9281-7c88a40d4b72","resolution":{"observed_at":"2026-08-12T14:14:09.214957Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-13T08:04:52.930897Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":80,"verified_exact":20,"verified_fuzzy":0},"total_outbound_references":147},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 100 of 147 outbound references and 0 inbound Pith citation observations for arXiv:2411.15634."}