{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:MVO7J72QXVCGWY3JN4PUFBOWEV","short_pith_number":"pith:MVO7J72Q","schema_version":"1.0","canonical_sha256":"655df4ff50bd446b63696f1f4285d62547a0985e686aa1f76e5f4818f148ee01","source":{"kind":"arxiv","id":"2307.12973","version":2},"attestation_state":"computed","paper":{"title":"Wisdom of Instruction-Tuned Language Model Crowds. Exploring Model Label Variation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Debora Nozza, Dirk Hovy, Flor Miriam Plaza-del-Arco","submitted_at":"2023-07-24T17:49:31Z","abstract_excerpt":"Large Language Models (LLMs) exhibit remarkable text classification capabilities, excelling in zero- and few-shot learning (ZSL and FSL) scenarios. However, since they are trained on different datasets, performance varies widely across tasks between those models. Recent studies emphasize the importance of considering human label variation in data annotation. However, how this human label variation also applies to LLMs remains unexplored. Given this likely model specialization, we ask: Do aggregate LLM labels improve over individual models (as for human annotators)? We evaluate four recent inst"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2307.12973","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-07-24T17:49:31Z","cross_cats_sorted":[],"title_canon_sha256":"39bd16e2c8e4ea2b173ac4e9a5457ae258f8c4cfa0055c4d2e6187bebf7c41cd","abstract_canon_sha256":"522407e52dfcbfd60245c8297e515c4eae657f91f8d7faa469154e096ea1e636"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:07:57.386267Z","signature_b64":"rD6zqCrrnVfX4bP2QJKb/yKy4kdVeSROTbI0j5TB1AKBe9tepDVAmR0zV/wfw6aw9LCxzKajMHMs8FgNGKrbCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"655df4ff50bd446b63696f1f4285d62547a0985e686aa1f76e5f4818f148ee01","last_reissued_at":"2026-07-05T08:07:57.385895Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:07:57.385895Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Wisdom of Instruction-Tuned Language Model Crowds. Exploring Model Label Variation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Debora Nozza, Dirk Hovy, Flor Miriam Plaza-del-Arco","submitted_at":"2023-07-24T17:49:31Z","abstract_excerpt":"Large Language Models (LLMs) exhibit remarkable text classification capabilities, excelling in zero- and few-shot learning (ZSL and FSL) scenarios. However, since they are trained on different datasets, performance varies widely across tasks between those models. Recent studies emphasize the importance of considering human label variation in data annotation. However, how this human label variation also applies to LLMs remains unexplored. Given this likely model specialization, we ask: Do aggregate LLM labels improve over individual models (as for human annotators)? We evaluate four recent inst"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.12973","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2307.12973/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2307.12973","created_at":"2026-07-05T08:07:57.385952+00:00"},{"alias_kind":"arxiv_version","alias_value":"2307.12973v2","created_at":"2026-07-05T08:07:57.385952+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.12973","created_at":"2026-07-05T08:07:57.385952+00:00"},{"alias_kind":"pith_short_12","alias_value":"MVO7J72QXVCG","created_at":"2026-07-05T08:07:57.385952+00:00"},{"alias_kind":"pith_short_16","alias_value":"MVO7J72QXVCGWY3J","created_at":"2026-07-05T08:07:57.385952+00:00"},{"alias_kind":"pith_short_8","alias_value":"MVO7J72Q","created_at":"2026-07-05T08:07:57.385952+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.18062","citing_title":"Security and Privacy Prompts in the Wild: What Users Ask LLMs and How LLMs Respond","ref_index":6,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV","json":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV.json","graph_json":"https://pith.science/api/pith-number/MVO7J72QXVCGWY3JN4PUFBOWEV/graph.json","events_json":"https://pith.science/api/pith-number/MVO7J72QXVCGWY3JN4PUFBOWEV/events.json","paper":"https://pith.science/paper/MVO7J72Q"},"agent_actions":{"view_html":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV","download_json":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV.json","view_paper":"https://pith.science/paper/MVO7J72Q","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2307.12973&json=true","fetch_graph":"https://pith.science/api/pith-number/MVO7J72QXVCGWY3JN4PUFBOWEV/graph.json","fetch_events":"https://pith.science/api/pith-number/MVO7J72QXVCGWY3JN4PUFBOWEV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV/action/storage_attestation","attest_author":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV/action/author_attestation","sign_citation":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV/action/citation_signature","submit_replication":"https://pith.science/pith/MVO7J72QXVCGWY3JN4PUFBOWEV/action/replication_record"}},"created_at":"2026-07-05T08:07:57.385952+00:00","updated_at":"2026-07-05T08:07:57.385952+00:00"}