{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:7DGYKJGB5LQ3JLUEYACBT7EG3S","short_pith_number":"pith:7DGYKJGB","canonical_record":{"source":{"id":"2410.13047","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-16T21:17:18Z","cross_cats_sorted":["cs.CL","cs.IR"],"title_canon_sha256":"43a038eb13d7a4b76b2dac9ab7665ad78b2c5ecfd52b8fca2a2dc3b4fa27a874","abstract_canon_sha256":"0e89a8104ca50cfef480419f26b7e40ccc1878b89324f69440334508ffffe404"},"schema_version":"1.0"},"canonical_sha256":"f8cd8524c1eae1b4ae84c00419fc86dcb5e18454294a2e246ddbedc9768709bd","source":{"kind":"arxiv","id":"2410.13047","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13047","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13047v2","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13047","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_12","alias_value":"7DGYKJGB5LQ3","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_16","alias_value":"7DGYKJGB5LQ3JLUE","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_8","alias_value":"7DGYKJGB","created_at":"2026-07-05T09:29:49Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:7DGYKJGB5LQ3JLUEYACBT7EG3S","target":"record","payload":{"canonical_record":{"source":{"id":"2410.13047","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-16T21:17:18Z","cross_cats_sorted":["cs.CL","cs.IR"],"title_canon_sha256":"43a038eb13d7a4b76b2dac9ab7665ad78b2c5ecfd52b8fca2a2dc3b4fa27a874","abstract_canon_sha256":"0e89a8104ca50cfef480419f26b7e40ccc1878b89324f69440334508ffffe404"},"schema_version":"1.0"},"canonical_sha256":"f8cd8524c1eae1b4ae84c00419fc86dcb5e18454294a2e246ddbedc9768709bd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:29:49.960281Z","signature_b64":"0/gVGAsi4CvHEKin/SgJJNofSVl5nfVcphf1ITMovlqhvP5aF/0Y3ADa7CkOOfIMWQnabEfCxFDrmjpYIgb9AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f8cd8524c1eae1b4ae84c00419fc86dcb5e18454294a2e246ddbedc9768709bd","last_reissued_at":"2026-07-05T09:29:49.959807Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:29:49.959807Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.13047","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Vz5hh7ssqLvmeBMRF/9oDMV3OUas9b552AD57hHDAdkZAAm1s6HVWEuX4tGg5m6DkpH6KXoLD3V3aQ1cnJx8Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T05:37:05.576087Z"},"content_sha256":"1c2018525f9e934c4e577dea3fa320f1eff7a061cf6a358056005a4cbe92b5e8","schema_version":"1.0","event_id":"sha256:1c2018525f9e934c4e577dea3fa320f1eff7a061cf6a358056005a4cbe92b5e8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:7DGYKJGB5LQ3JLUEYACBT7EG3S","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLM Confidence Evaluation Measures in Zero-Shot CSS Classification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.IR"],"primary_cat":"cs.HC","authors_text":"David Farr, Iain Cruickshank, Jevin West, Kate Starbird, Nicholas Clark, Nico Manzonelli","submitted_at":"2024-10-16T21:17:18Z","abstract_excerpt":"Assessing classification confidence is critical for leveraging large language models (LLMs) in automated labeling tasks, especially in the sensitive domains presented by Computational Social Science (CSS) tasks. In this paper, we make three key contributions: (1) we propose an uncertainty quantification (UQ) performance measure tailored for data annotation tasks, (2) we compare, for the first time, five different UQ strategies across three distinct LLMs and CSS data annotation tasks, (3) we introduce a novel UQ aggregation strategy that effectively identifies low-confidence LLM annotations and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13047","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.13047/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mpNUyue8xqUqkpS91rFI3y3u1RPI72mrpSg4y224bALLsFmuaOa2oVYp1qlMWflkXPZaXXL4nKpkwyuFB0hbAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T05:37:05.576658Z"},"content_sha256":"e95f159773c264dc0f8f5fbc3617ecd22b03e65f0e2ac028c8371778ca6e943f","schema_version":"1.0","event_id":"sha256:e95f159773c264dc0f8f5fbc3617ecd22b03e65f0e2ac028c8371778ca6e943f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/bundle.json","state_url":"https://pith.science/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T05:37:05Z","links":{"resolver":"https://pith.science/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S","bundle":"https://pith.science/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/bundle.json","state":"https://pith.science/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7DGYKJGB5LQ3JLUEYACBT7EG3S/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:7DGYKJGB5LQ3JLUEYACBT7EG3S","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0e89a8104ca50cfef480419f26b7e40ccc1878b89324f69440334508ffffe404","cross_cats_sorted":["cs.CL","cs.IR"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-16T21:17:18Z","title_canon_sha256":"43a038eb13d7a4b76b2dac9ab7665ad78b2c5ecfd52b8fca2a2dc3b4fa27a874"},"schema_version":"1.0","source":{"id":"2410.13047","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13047","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13047v2","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13047","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_12","alias_value":"7DGYKJGB5LQ3","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_16","alias_value":"7DGYKJGB5LQ3JLUE","created_at":"2026-07-05T09:29:49Z"},{"alias_kind":"pith_short_8","alias_value":"7DGYKJGB","created_at":"2026-07-05T09:29:49Z"}],"graph_snapshots":[{"event_id":"sha256:e95f159773c264dc0f8f5fbc3617ecd22b03e65f0e2ac028c8371778ca6e943f","target":"graph","created_at":"2026-07-05T09:29:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.13047/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Assessing classification confidence is critical for leveraging large language models (LLMs) in automated labeling tasks, especially in the sensitive domains presented by Computational Social Science (CSS) tasks. In this paper, we make three key contributions: (1) we propose an uncertainty quantification (UQ) performance measure tailored for data annotation tasks, (2) we compare, for the first time, five different UQ strategies across three distinct LLMs and CSS data annotation tasks, (3) we introduce a novel UQ aggregation strategy that effectively identifies low-confidence LLM annotations and","authors_text":"David Farr, Iain Cruickshank, Jevin West, Kate Starbird, Nicholas Clark, Nico Manzonelli","cross_cats":["cs.CL","cs.IR"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-16T21:17:18Z","title":"LLM Confidence Evaluation Measures in Zero-Shot CSS Classification"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13047","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1c2018525f9e934c4e577dea3fa320f1eff7a061cf6a358056005a4cbe92b5e8","target":"record","created_at":"2026-07-05T09:29:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0e89a8104ca50cfef480419f26b7e40ccc1878b89324f69440334508ffffe404","cross_cats_sorted":["cs.CL","cs.IR"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-16T21:17:18Z","title_canon_sha256":"43a038eb13d7a4b76b2dac9ab7665ad78b2c5ecfd52b8fca2a2dc3b4fa27a874"},"schema_version":"1.0","source":{"id":"2410.13047","kind":"arxiv","version":2}},"canonical_sha256":"f8cd8524c1eae1b4ae84c00419fc86dcb5e18454294a2e246ddbedc9768709bd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f8cd8524c1eae1b4ae84c00419fc86dcb5e18454294a2e246ddbedc9768709bd","first_computed_at":"2026-07-05T09:29:49.959807Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:29:49.959807Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0/gVGAsi4CvHEKin/SgJJNofSVl5nfVcphf1ITMovlqhvP5aF/0Y3ADa7CkOOfIMWQnabEfCxFDrmjpYIgb9AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:29:49.960281Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.13047","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1c2018525f9e934c4e577dea3fa320f1eff7a061cf6a358056005a4cbe92b5e8","sha256:e95f159773c264dc0f8f5fbc3617ecd22b03e65f0e2ac028c8371778ca6e943f"],"state_sha256":"6193314c897538d99035a841013e8a25bb5dded4c22bdad9a5a7bc3f0ea7d9b6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4Xuwr/aK1xIz62qMddW0Yftj02uWfdkrdNNIkFFpL+zKgEFwelhq6SMVe8j/Yy/0Po9SAQvakTMYDRGaWYTSBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T05:37:05.581719Z","bundle_sha256":"ab6f9baf1eaa3275001dcd5b1992535ac7c0bdf00d79f177d32877bf98112414"}}