{"as_of":"2026-08-13T04:27:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d54710c0641f6c022bbc85ec01d2390134735b9318fbce53826fe3efd4d6b649","coverage":[{"denominator":48,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":48,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T15:58:47.648449Z","state":"measured"},{"denominator":48,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":48,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.13760/citation-record","integrity":"/paper/2411.13760/integrity","json":"/paper/2411.13760/citation-record.json","paper":"/paper/2411.13760"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.833653Z","title":"DICES Dataset: Diversity in Conversational AI Evaluation for Safety","venue":null,"work_id":"deee1d94-7f46-4d40-ab07-38954a7328be","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.410920Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:8d67c756dbfeb130689afa90084c3d9278792bd406995c0a2e3dad2997100a25","observation_id":"5c077c67-c093-4ebf-88d3-ecb29673ac03","resolution":{"observed_at":"2026-08-12T15:58:48.839258Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.16133","last_updated":"2022-11-30T09:15:10Z","snapshot_observed_at":"2026-08-12T23:53:19.450185Z","submitted_at":"2022-10-28T14:01:32Z","title":"Stop Measuring Calibration When Humans Disagree","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.16133","snapshot_observed_at":"2026-08-12T15:58:47.416548Z","title":"Stop Measuring Calibration When Humans Disagree, November 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.416548Z"},"links":{"cited_paper":"/paper/2210.16133","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:6b576ee7a81cc72417afe8ef5f722214e42f156bdba58f5e455a145415bc8a06","observation_id":"a57fd76d-a59b-4874-9ec1-70b47786c546","resolution":{"observed_at":"2026-08-12T15:58:47.416548Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.814957Z","title":"It’s the End of the Gold Standard as we Know it","venue":null,"work_id":"12bc3831-cebe-4f10-bad6-627305208a1d","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.422176Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:a0b09546be6f764e80bcdb6900dc5b20a68c19aae1af85fa491539889a8e1ec3","observation_id":"2b3da26a-e236-43ed-9e86-cac277deb601","resolution":{"observed_at":"2026-08-12T15:58:48.821568Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1707.01477","last_updated":"2017-07-05T17:19:45Z","snapshot_observed_at":"2026-07-06T05:49:51.875694Z","submitted_at":"2017-07-05T17:19:45Z","title":"Like trainer, like bot? Inheritance of bias in algorithmic content moderation","version":1},"cited_work":{"arxiv_id":"1707.01477","doi":null,"metadata_source":"pith","pith_arxiv_id":"1707.01477","snapshot_observed_at":"2026-08-12T15:58:48.513949Z","title":"Like trainer, like bot? Inheritance of bias in algorithmic content moderation","venue":"cs.CY","work_id":"6b8f95ff-d5bb-4e80-9c7f-d0f847d436d0","year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.427557Z"},"links":{"cited_paper":"/paper/1707.01477","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:19c65030c8ba7886632d0536dfde786fb85ad3d8cdfb42ca283a4ad526b79ddf","observation_id":"8d37a0ce-5096-4610-a17a-d9fba6cdd5c2","resolution":{"observed_at":"2026-08-12T15:58:48.520699Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.797194Z","title":"Holistic evaluation of language models","venue":null,"work_id":"b893e7dc-0314-46ce-bfab-17271f7580b8","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.433122Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:6c10fdca16b8cb59669dc3377e5b5805b8ab75b7b54b5ad88c8409924e9051c3","observation_id":"28c48695-e4b3-47dc-8a24-ac5d9e0ce8b6","resolution":{"observed_at":"2026-08-12T15:58:48.802595Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.438380Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.438380Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:7d96710c472705b429f7adedcf3e5f1cbeade4db08134d0d9f6f71e88df1b46c","observation_id":"28671047-f62c-4e76-bf92-a379645b173b","resolution":{"observed_at":"2026-08-12T15:58:47.438380Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.447529Z","title":"Revolt: Collaborative Crowdsourcing for Labeling Machine Learning Datasets","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.447529Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:ca87952d8fb033728c5de64fb19c0d95aea1a1bd913ff8b6aa4aab5d3750344a","observation_id":"00d71071-c431-4e74-8027-1c47e9925021","resolution":{"observed_at":"2026-08-12T15:58:47.447529Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.780952Z","title":"Judgment sieve: Reducing uncertainty in group judgments through interventions targeting ambiguity versus disagreement","venue":null,"work_id":"3dfa5c3f-9ec1-4407-871b-59c7cc02b7e8","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.453452Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:94d51b4cb627f90a0e06f2ac9c7cd5cc9910643dfd7b5613e709309416176b70","observation_id":"c6af6ccc-9894-468b-b70a-b43f38dc4034","resolution":{"observed_at":"2026-08-12T15:58:48.786308Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.01615","last_updated":"2023-05-02T17:17:50Z","snapshot_observed_at":"2026-08-13T04:12:01.523788Z","submitted_at":"2023-05-02T17:17:50Z","title":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","version":1},"cited_work":{"arxiv_id":"2305.01615","doi":null,"metadata_source":"pith","pith_arxiv_id":"2305.01615","snapshot_observed_at":"2026-08-12T15:58:48.351605Z","title":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","venue":"cs.HC","work_id":"74725ac7-1a15-404f-af56-04e7f7557c2a","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.459514Z"},"links":{"cited_paper":"/paper/2305.01615","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:0500983f73e3a683cd5cd606dfef3de47ba3f50698c982b7072bdf1df8ea40aa","observation_id":"9a458d73-d10a-4790-80a3-159d996761a7","resolution":{"observed_at":"2026-08-12T15:58:48.356776Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1162/tacl_a_00449/","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.787401Z","title":"Dealing with Disagree- ments: Looking Beyond the Majority V ote in Subjective Annotations","venue":null,"work_id":"c16a0bc0-ad6b-46bb-80a2-4769600dcbbf","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.465135Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9c851f70da182c5a52491c78f3292b6a7aa1a0bdde76a06c6ddc1e4a2a87abcf","observation_id":"9b8ed86a-925b-4376-959a-e23aace9ef7d","resolution":{"observed_at":"2026-08-12T15:58:47.793619Z","resolver_source":"doi_truncated","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.10857","last_updated":"2024-04-16T19:12:03Z","snapshot_observed_at":"2026-08-13T00:28:51.239715Z","submitted_at":"2024-04-16T19:12:03Z","title":"D3CODE: Disentangling Disagreements in Data across Cultures on Offensiveness Detection and Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.10857","snapshot_observed_at":"2026-08-12T15:58:47.470355Z","title":"D3CODE: Disentan- gling Disagreements in Data across Cultures on Offensiveness Detection and Evaluation, April 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.470355Z"},"links":{"cited_paper":"/paper/2404.10857","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:812c8f5df87f0e8e9f6693c0c96814b9266e1995ce03f96e090e4a75de37ca6b","observation_id":"ff854f86-582a-476c-b805-f332c7290525","resolution":{"observed_at":"2026-08-12T15:58:47.470355Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.15897","last_updated":"2024-08-27T18:23:44Z","snapshot_observed_at":"2026-08-11T13:51:41.602934Z","submitted_at":"2024-01-29T05:46:14Z","title":"Red-Teaming for Generative AI: Silver Bullet or Security Theater?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.15897","snapshot_observed_at":"2026-08-12T15:58:47.475898Z","title":"Red-teaming for generative ai: Silver bullet or security theater? arXiv preprint arXiv:2401.15897, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.475898Z"},"links":{"cited_paper":"/paper/2401.15897","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:7b92f3a8d24ad03841c868c1b88a37643101dac0ef9dccf9a8f35fab40c3c274","observation_id":"3ea58abb-d693-4056-aac1-58b585a5ad0a","resolution":{"observed_at":"2026-08-12T15:58:47.475898Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2007.03114","last_updated":"2021-02-02T06:29:04Z","snapshot_observed_at":"2026-08-07T09:30:47.028966Z","submitted_at":"2020-07-06T23:13:07Z","title":"Efficient Conformal Prediction via Cascaded Inference with Expanded Admission","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2007.03114","snapshot_observed_at":"2026-08-12T15:58:47.481439Z","title":"Efficient conformal prediction via cascaded inference with expanded admission","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.481439Z"},"links":{"cited_paper":"/paper/2007.03114","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:16464f3f7d8ba86f82ff99c0d681d71b54d866e3d7a178771743357f52930e08","observation_id":"8aed3702-9d7b-445e-aefd-781624d01b0f","resolution":{"observed_at":"2026-08-12T15:58:47.481439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.05860","last_updated":"2024-05-09T15:48:07Z","snapshot_observed_at":"2026-08-13T00:10:53.019540Z","submitted_at":"2024-05-09T15:48:07Z","title":"The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.05860","snapshot_observed_at":"2026-08-12T15:58:47.486664Z","title":"The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels, May 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.486664Z"},"links":{"cited_paper":"/paper/2405.05860","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:670a8114cd7cbcbe4a9994d1ae80fda074adb06014ebc889dff2f17c1621258f","observation_id":"0ea16206-678b-402d-8ee8-379a7cd8505f","resolution":{"observed_at":"2026-08-12T15:58:47.486664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.07858","last_updated":"2022-11-22T19:12:57Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-08-23T23:37:14Z","title":"Red Teaming Language Models to Reduce Harms: Methods, Scaling Behaviors, and Lessons Learned","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.07858","snapshot_observed_at":"2026-08-12T15:58:47.491762Z","title":"Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.491762Z"},"links":{"cited_paper":"/paper/2209.07858","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:c593d14c3f67193121fb6f05988fe4f9e72016d1254ea1c3003a4cbc5e5f5f12","observation_id":"cb126527-a912-4f1a-a3e6-558a7fc321a7","resolution":{"observed_at":"2026-08-12T15:58:47.491762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2017.26899","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.234164Z","title":"Deep Label Distribution Learning With Label Ambiguity","venue":null,"work_id":"ea751607-5866-47a6-b26e-48f00a086360","year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.496224Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:a8445c43a36627d6fada49eb294010c3cbc25a3b5809215740fe762f0424db68","observation_id":"68160aa2-5850-4122-90bc-0f1ae0396ae7","resolution":{"observed_at":"2026-08-12T15:58:48.243981Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.500370Z","title":"Label Distribution Learning","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.500370Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:7abdd97c7be9dacdb6ced5422846f15a392b79f895eeb7a4c0008025c0796dcd","observation_id":"4809e15c-34b0-4d78-b633-a8e1ed9dc364","resolution":{"observed_at":"2026-08-12T15:58:47.500370Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.764392Z","title":"The disagreement deconvolution: Bringing machine learning performance metrics in line with reality","venue":null,"work_id":"950771a7-1b5e-4d6e-a50a-6f2505e18bca","year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.504721Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:7c081d4a452488cbfd56af245dc992befefd153d9ea2839513ead5a382565880","observation_id":"210bc53a-7606-4252-9fbc-38252d34451d","resolution":{"observed_at":"2026-08-12T15:58:48.769904Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.508867Z","title":"Gordon, Kaitlyn Zhou, Kayur Patel, Tatsunori Hashimoto, and Michael S","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.508867Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9486cf2e5b8043f57803617ae5620d69bbf2d716f1863c637fc1656896d86874","observation_id":"a7959e7b-177f-4f5d-a252-6e4a117e8c02","resolution":{"observed_at":"2026-08-12T15:58:47.508867Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.746148Z","title":"Jury learning: Integrating dissenting voices into machine learning models","venue":null,"work_id":"4b3b4df3-7f23-4390-95d1-e9bca9e08aca","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.513299Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:e247660ff881d665f3805bdc0328de2bd3cb343b0960255decf96db6ecb861ca","observation_id":"a7d6d6bd-d8b1-45c2-9a75-a31e0516dfad","resolution":{"observed_at":"2026-08-12T15:58:48.752362Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.729244Z","title":"Is your toxicity my toxicity? exploring the impact of rater identity on toxicity annotation","venue":null,"work_id":"06ae60f1-8c81-4cbf-a6b6-547ee10b2434","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.517490Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:3e9ba7e3d5023f8acf57232f9314e7c708d4cf99c3db37a11790e33eb7efaa42","observation_id":"9cbcbbe0-069c-473b-9568-f8cd5de417a1","resolution":{"observed_at":"2026-08-12T15:58:48.734640Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-10T12:35:09.020030Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-12T15:58:47.522165Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.522165Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9448e1a26a4161ae9f53052b069ff9699fa850cd3bb1f326aa5ce6b34ae75d66","observation_id":"50d372cd-4d44-4a5d-aa16-ca30d7b7bb7b","resolution":{"observed_at":"2026-08-12T15:58:47.522165Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.711218Z","title":"Intersectionality in ai safety: Using multilevel models to understand diverse perceptions of safety in conversational ai","venue":null,"work_id":"d5117602-5332-4a1a-a918-c8c6a380e8de","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.527335Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:6802010f4dd3428413bc476b7b00c4ec7810e9cb8a2d68492ecf5ede189c1133","observation_id":"28b35d4a-d867-48bc-9e37-4ffe9a43f1d8","resolution":{"observed_at":"2026-08-12T15:58:48.717943Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.531722Z","title":"Culturally Aware Natural Language Inference","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.531722Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:d4db82c73c1b445bf59d8950c82901b96a4145263fa6acda4be48354a7462a7a","observation_id":"1bf6244c-5633-479d-990b-713b16189533","resolution":{"observed_at":"2026-08-12T15:58:47.531722Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.694012Z","title":"Chaithanya Manam, Dwarakanath Jampani, Mariam Zaim, Meng-Han Wu, and Alexander J","venue":null,"work_id":"81e3c1c8-c4ee-44d6-8be1-60e290fdd1ad","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.536697Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:35c115edba22e707a03be5f3fbe846af5cc1c38bfdbaa097e74dccbb666f5041","observation_id":"77dd0d86-8374-4649-9493-e5226e7d39dd","resolution":{"observed_at":"2026-08-12T15:58:48.699453Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.541649Z","title":"Annotation Error Detection: Analyzing the Past and Present for a More Coherent Future","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.541649Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:e4fa32845081b1fb1d433070684762030e99e5a4793822010d0fd9f7282e9022","observation_id":"a874b2e7-e4c3-487c-88ed-ec1d04b97ef8","resolution":{"observed_at":"2026-08-12T15:58:47.541649Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.546651Z","title":"A Bayesian Framework for Modeling Human Evaluations","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.546651Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:c9a0749ee8bfe0ae4e05e1f28802ba688eab3d032953fd5f3815d6ef5bd278a2","observation_id":"d383fbc1-768b-4684-90d8-75a83dca527e","resolution":{"observed_at":"2026-08-12T15:58:47.546651Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.672946Z","title":"Reconsidering Annotator Disagreement about Racist Language: Noise or Signal?","venue":null,"work_id":"eb19a687-cd6d-4c49-b6ea-a5877b6f57fe","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.552442Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:bbf80840df22df60175894a2e897b0f49e6fb2089334142ec55ae2bb493690f3","observation_id":"d3ef8992-3d96-46b0-ad4e-d464487125d0","resolution":{"observed_at":"2026-08-12T15:58:48.679755Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.655723Z","title":"Learning to predict population-level label distributions","venue":null,"work_id":"7e31b0be-9973-417e-9a45-7218315032e0","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.557193Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:c9d0a0227ab927e07f1f9e130c5778dcccbc8505e684f9d3c9a0394703bb9422","observation_id":"c16a2fd9-fa2f-488f-aada-166b24f23f4d","resolution":{"observed_at":"2026-08-12T15:58:48.661299Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04893","last_updated":"2024-03-07T20:55:08Z","snapshot_observed_at":"2026-08-13T00:59:45.170564Z","submitted_at":"2024-03-07T20:55:08Z","title":"A Safe Harbor for AI Evaluation and Red Teaming","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04893","snapshot_observed_at":"2026-08-12T15:58:47.561742Z","title":"A safe harbor for ai evaluation and red teaming","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.561742Z"},"links":{"cited_paper":"/paper/2403.04893","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:24b374cc3ff4ab768cc841c216056374e07539dc023e0bb791653c13cfad5187","observation_id":"f5c68c08-3d94-4613-8b4f-72dae82cb0c8","resolution":{"observed_at":"2026-08-12T15:58:47.561742Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17750","last_updated":"2023-10-26T19:45:06Z","snapshot_observed_at":"2026-08-10T20:14:16.170396Z","submitted_at":"2023-10-26T19:45:06Z","title":"A Framework for Automated Measurement of Responsible AI Harms in Generative AI Applications","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17750","snapshot_observed_at":"2026-08-12T15:58:47.566997Z","title":"A framework for automated measurement of responsible ai harms in generative ai applications","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.566997Z"},"links":{"cited_paper":"/paper/2310.17750","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:3aeef417ccdff7e238ff05d0f2342ca9a8e9174b3e424735d9622f7091201188","observation_id":"b032cac8-7e8e-4f0f-ac4b-c349810d2c66","resolution":{"observed_at":"2026-08-12T15:58:47.566997Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1609/hcomp.v6i1.13338","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.708020Z","title":null,"venue":null,"work_id":"32302c9f-78da-4ed9-967e-146df77b1f90","year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.572311Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:b87a5117b9cd65801485d2eee0128d31febb8dfcfb90de5efd75939cd6a93506","observation_id":"e9a420e2-695e-46aa-97db-9bd5713d93c7","resolution":{"observed_at":"2026-08-12T15:58:47.716100Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2004.09456","last_updated":"2020-04-20T17:14:33Z","snapshot_observed_at":"2026-07-06T09:13:46.854094Z","submitted_at":"2020-04-20T17:14:33Z","title":"StereoSet: Measuring stereotypical bias in pretrained language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.09456","snapshot_observed_at":"2026-08-12T15:58:47.577191Z","title":"Stereoset: Measuring stereotypical bias in pretrained language models","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.577191Z"},"links":{"cited_paper":"/paper/2004.09456","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:cfe1cf243f8e8095c3874b50ad886c083c2a722a2eb5f1ba2c8460827d0352b3","observation_id":"630a4f9b-ddfe-4f13-bd99-180ec7bc36f4","resolution":{"observed_at":"2026-08-12T15:58:47.577191Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.638907Z","title":"Diversity-aware annotation for conversational ai safety","venue":null,"work_id":"b178a89e-dd2c-4e14-b400-0273f87dfa42","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.581750Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:3fc90fa98d49938629eaccdd6cf1ddfe6f809d0fd458ea467fa37479069b5962","observation_id":"6657a046-e9a1-4318-a733-e6ca4ee0f37e","resolution":{"observed_at":"2026-08-12T15:58:48.645028Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.622287Z","title":"Inherent disagreements in human textual inferences","venue":null,"work_id":"97c0e4f2-b05f-4896-ae43-fd1f51b5a1fc","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.586220Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:73854e0eb65c3f223d2314fdc32ee9a0f546266284750dca4910517182988c14","observation_id":"a6e9d212-c5aa-486e-ac2b-6bb52305a3f3","resolution":{"observed_at":"2026-08-12T15:58:48.627575Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.591125Z","title":"Inherent Disagreements in Human Textual Inferences","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.591125Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:31d38a26b3de58c723f73e9e091a1bd1a9e5a638b11c6dc3054feec6a2661f54","observation_id":"4540f88d-e957-4faf-8bf6-61f13e4b7019","resolution":{"observed_at":"2026-08-12T15:58:47.591125Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.605626Z","title":"Human uncertainty makes classification more robust","venue":null,"work_id":"f3b31b87-2b79-402e-bb28-c75d0de90932","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.595776Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:d23a5691d0015b909cb416eb2ecad753572d1db809569e88e86cdc8bf5924fbc","observation_id":"ea6b370c-3186-416d-83f1-dc46594621fe","resolution":{"observed_at":"2026-08-12T15:58:48.610984Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.02570","last_updated":"2022-11-04T16:38:09Z","snapshot_observed_at":"2026-08-06T05:59:11.266228Z","submitted_at":"2022-11-04T16:38:09Z","title":"The 'Problem' of Human Label Variation: On Ground Truth in Data, Modeling and Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.02570","snapshot_observed_at":"2026-08-12T15:58:47.600485Z","title":"The ’Problem’ of Human Label Variation: On Ground Truth in Data, Modeling and Evaluation, November 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.600485Z"},"links":{"cited_paper":"/paper/2211.02570","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:56fbae162546a3a758695eda8af271f24d5f9821bc79e206d30b9399b73d064f","observation_id":"8c67e5e5-81c0-44dc-87a4-d46a433706d4","resolution":{"observed_at":"2026-08-12T15:58:47.600485Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.05699","last_updated":"2021-10-12T02:35:45Z","snapshot_observed_at":"2026-08-12T15:43:01.721867Z","submitted_at":"2021-10-12T02:35:45Z","title":"On Releasing Annotator-Level Labels and Information in Datasets","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.05699","snapshot_observed_at":"2026-08-12T15:58:47.605264Z","title":"On Releasing Annotator- Level Labels and Information in Datasets, October 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.605264Z"},"links":{"cited_paper":"/paper/2110.05699","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:b770e9cd1bf258d9daa57188e1423bf76b3b2460c23b36626de3187746b74348","observation_id":"272987e6-d92a-44e6-a2a4-c316535e7c9a","resolution":{"observed_at":"2026-08-12T15:58:47.605264Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.01254","last_updated":"2026-04-22T19:47:51Z","snapshot_observed_at":"2026-07-06T11:15:17.300144Z","submitted_at":"2021-06-02T16:07:32Z","title":"Principled Evaluation with Human Labels: One Rater at a Time and Rater Equivalence","version":3},"cited_work":{"arxiv_id":"2106.01254","doi":null,"metadata_source":"pith","pith_arxiv_id":"2106.01254","snapshot_observed_at":"2026-08-12T15:58:47.865516Z","title":"Principled Evaluation with Human Labels: One Rater at a Time and Rater Equivalence","venue":"cs.LG","work_id":"b1e344dc-7442-45f9-bfbe-9bd477c4fa46","year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.610350Z"},"links":{"cited_paper":"/paper/2106.01254","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:cff163f88877215f1f6fdfcb1051c486bd35849ab9fae3f82f51066c1c23ca4c","observation_id":"f1e7394c-d3f0-43e8-b506-d192bc39d250","resolution":{"observed_at":"2026-08-12T15:58:47.871195Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2111.07997","last_updated":"2022-05-09T23:58:07Z","snapshot_observed_at":"2026-07-06T12:08:46.570529Z","submitted_at":"2021-11-15T18:58:20Z","title":"Annotators with Attitudes: How Annotator Beliefs And Identities Bias Toxic Language Detection","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.07997","snapshot_observed_at":"2026-08-12T15:58:47.615404Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.615404Z"},"links":{"cited_paper":"/paper/2111.07997","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:b5d79075e836b76af42c56d9635a8d5aabe4f67f5763d11e89e94f0d92e0548d","observation_id":"602c5eaf-4359-4f96-8eaa-e3f5f29ae945","resolution":{"observed_at":"2026-08-12T15:58:47.615404Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12272","last_updated":"2024-04-18T15:45:27Z","snapshot_observed_at":"2026-08-13T00:27:18.458669Z","submitted_at":"2024-04-18T15:45:27Z","title":"Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12272","snapshot_observed_at":"2026-08-12T15:58:47.620627Z","title":"Who validates the validators? aligning llm-assisted evaluation of llm outputs with human preferences","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.620627Z"},"links":{"cited_paper":"/paper/2404.12272","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:05535b28b7a49c32c26a183d2e4a3f460266fc683778b2816d4a3b6fd3675472","observation_id":"09088a7d-bbd7-4f5d-ac12-f8a906a710b3","resolution":{"observed_at":"2026-08-12T15:58:47.620627Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.625408Z","title":"A case for soft loss functions","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.625408Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:580b6d6d815a6bf2a6cd1d17edd23c83539fd02cef8695db1ec5aa0dc0129684","observation_id":"83efe431-4f75-4d3f-a57a-9139d956405b","resolution":{"observed_at":"2026-08-12T15:58:47.625408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1804.07461","last_updated":"2019-02-22T23:53:34Z","snapshot_observed_at":"2026-07-06T06:34:26.609892Z","submitted_at":"2018-04-20T06:35:04Z","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1804.07461","snapshot_observed_at":"2026-08-12T15:58:47.630144Z","title":"Glue: A multi-task benchmark and analysis platform for natural language understanding","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.630144Z"},"links":{"cited_paper":"/paper/1804.07461","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:1e7238a875e62de4c89446cc00026b3e76b14cb844e3d1561ba28b7028c4fb34","observation_id":"e5c52f95-88a5-46ce-9d23-698238ed1636","resolution":{"observed_at":"2026-08-12T15:58:47.630144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.577470Z","title":"gold data","venue":null,"work_id":"626d2fbc-5cb6-4c10-ba16-e614c3ce753b","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.634597Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:4b123d93af1429e2b0449dfb351f025d11fc16e8ef08d6a4fd24ae5bbcb2d5e5","observation_id":"affa9a7a-e8f2-410a-a2c9-bcd63e415c00","resolution":{"observed_at":"2026-08-12T15:58:48.582824Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.638882Z","title":"Super- naturalinstructions:generalization via declarative instructions on 1600+ tasks","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.638882Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:2e5c99a87393d923e7367d25167c6999019724c978e995ff3bca71e0705aa4c3","observation_id":"993a6c94-0d98-48a6-ac30-d1adeb6b00b7","resolution":{"observed_at":"2026-08-12T15:58:47.638882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.643852Z","title":"Disagreement Matters: Preserving Label Diversity by Jointly Modeling Item and Anno- tator Label Distributions with DisCo","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.643852Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:5d4249a5518f36721d0710fcdde1f28430bf01625e9dba1893f5b478b9918a1e","observation_id":"86907fa7-da6a-4695-adfa-7f5c27498499","resolution":{"observed_at":"2026-08-12T15:58:47.643852Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.551171Z","title":"Many islands, many problems: An empirical examination of online safety behaviors in the caribbean","venue":null,"work_id":"f4945d3c-e9e7-4803-acd3-954af7aa0cda","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.648449Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:dfd91e5354c29f268f4ecca244e8a4267a34d973fbb31a73ea4e1aad4fb0b33c","observation_id":"35c8acda-de24-46ca-a0e7-f120e657f2f7","resolution":{"observed_at":"2026-08-12T15:58:48.555782Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-12T15:52:45.031684Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy"},"reference_resolution":{"displayed":48,"state_counts":{"malformed_identifier":3,"metadata_mismatch":3,"parse_uncertain":0,"unresolved":24,"verified_exact":2,"verified_fuzzy":16},"total_outbound_references":48},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 48 of 48 outbound references and 0 inbound Pith citation observations for arXiv:2411.13760."}