{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:JEKWLGBA7AMDLRPRIHU5W37LWB","short_pith_number":"pith:JEKWLGBA","canonical_record":{"source":{"id":"2310.15638","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-24T08:56:49Z","cross_cats_sorted":[],"title_canon_sha256":"0e9e22e1e1019cbf27720a22238591ac0f799ae5bc86571b0ab4a33ad25926fb","abstract_canon_sha256":"5e56ea8152bcfb4ec9fb0d3ed700cbd0acd507ddccf5e67c234ccf8cafc924f0"},"schema_version":"1.0"},"canonical_sha256":"4915659820f81835c5f141e9db6febb0470684b9f4a8cbd8b41a89a50dd341cf","source":{"kind":"arxiv","id":"2310.15638","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.15638","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"arxiv_version","alias_value":"2310.15638v1","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.15638","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_12","alias_value":"JEKWLGBA7AMD","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_16","alias_value":"JEKWLGBA7AMDLRPR","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_8","alias_value":"JEKWLGBA","created_at":"2026-07-05T07:56:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:JEKWLGBA7AMDLRPRIHU5W37LWB","target":"record","payload":{"canonical_record":{"source":{"id":"2310.15638","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-24T08:56:49Z","cross_cats_sorted":[],"title_canon_sha256":"0e9e22e1e1019cbf27720a22238591ac0f799ae5bc86571b0ab4a33ad25926fb","abstract_canon_sha256":"5e56ea8152bcfb4ec9fb0d3ed700cbd0acd507ddccf5e67c234ccf8cafc924f0"},"schema_version":"1.0"},"canonical_sha256":"4915659820f81835c5f141e9db6febb0470684b9f4a8cbd8b41a89a50dd341cf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:56:18.164202Z","signature_b64":"T8KolUk47LpEVNOBNEkGnwWzjmSkm65A6eJuuxarb/jNwpO5HQMa5R0DnlWCsHFM+a5/eAZE7AbALfnyEDdZCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4915659820f81835c5f141e9db6febb0470684b9f4a8cbd8b41a89a50dd341cf","last_reissued_at":"2026-07-05T07:56:18.163699Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:56:18.163699Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.15638","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JTLdnoRzWQqhKHVzKu7Gut74vO292TqrOTUVw9BOreS72E28oxMAt/1qo/QZa5+4kK7TlsBWWZkknnS4CXevBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T09:37:59.063930Z"},"content_sha256":"51a3f4b02e05dcbd3c2a30240f3618de083257b8f9701c04d1cd60abdf3b581c","schema_version":"1.0","event_id":"sha256:51a3f4b02e05dcbd3c2a30240f3618de083257b8f9701c04d1cd60abdf3b581c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:JEKWLGBA7AMDLRPRIHU5W37LWB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CoAnnotating: Uncertainty-Guided Work Allocation between Human and Large Language Models for Data Annotation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Caleb Ziems, Diyi Yang, Min-Yen Kan, Minzhi Li, Nancy F. Chen, Taiwei Shi, Zhengyuan Liu","submitted_at":"2023-10-24T08:56:49Z","abstract_excerpt":"Annotated data plays a critical role in Natural Language Processing (NLP) in training models and evaluating their performance. Given recent developments in Large Language Models (LLMs), models such as ChatGPT demonstrate zero-shot capability on many text-annotation tasks, comparable with or even exceeding human annotators. Such LLMs can serve as alternatives for manual annotation, due to lower costs and higher scalability. However, limited work has leveraged LLMs as complementary annotators, nor explored how annotation work is best allocated among humans and LLMs to achieve both quality and co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.15638","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.15638/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ygTWIeUVP/tSfqrr27EyvRN5nG1CwHrrrANeKjO9BcpIAH+0QJMGtchgHoJLKEv9Qk1BkTqmea0iuSncHdBfBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T09:37:59.064401Z"},"content_sha256":"38d35515bf7b58b0c1b12e5f393b1da29e9734a233ed4118d51397249ec71d17","schema_version":"1.0","event_id":"sha256:38d35515bf7b58b0c1b12e5f393b1da29e9734a233ed4118d51397249ec71d17"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/bundle.json","state_url":"https://pith.science/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T09:37:59Z","links":{"resolver":"https://pith.science/pith/JEKWLGBA7AMDLRPRIHU5W37LWB","bundle":"https://pith.science/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/bundle.json","state":"https://pith.science/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JEKWLGBA7AMDLRPRIHU5W37LWB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:JEKWLGBA7AMDLRPRIHU5W37LWB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5e56ea8152bcfb4ec9fb0d3ed700cbd0acd507ddccf5e67c234ccf8cafc924f0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-24T08:56:49Z","title_canon_sha256":"0e9e22e1e1019cbf27720a22238591ac0f799ae5bc86571b0ab4a33ad25926fb"},"schema_version":"1.0","source":{"id":"2310.15638","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.15638","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"arxiv_version","alias_value":"2310.15638v1","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.15638","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_12","alias_value":"JEKWLGBA7AMD","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_16","alias_value":"JEKWLGBA7AMDLRPR","created_at":"2026-07-05T07:56:18Z"},{"alias_kind":"pith_short_8","alias_value":"JEKWLGBA","created_at":"2026-07-05T07:56:18Z"}],"graph_snapshots":[{"event_id":"sha256:38d35515bf7b58b0c1b12e5f393b1da29e9734a233ed4118d51397249ec71d17","target":"graph","created_at":"2026-07-05T07:56:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.15638/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Annotated data plays a critical role in Natural Language Processing (NLP) in training models and evaluating their performance. Given recent developments in Large Language Models (LLMs), models such as ChatGPT demonstrate zero-shot capability on many text-annotation tasks, comparable with or even exceeding human annotators. Such LLMs can serve as alternatives for manual annotation, due to lower costs and higher scalability. However, limited work has leveraged LLMs as complementary annotators, nor explored how annotation work is best allocated among humans and LLMs to achieve both quality and co","authors_text":"Caleb Ziems, Diyi Yang, Min-Yen Kan, Minzhi Li, Nancy F. Chen, Taiwei Shi, Zhengyuan Liu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-24T08:56:49Z","title":"CoAnnotating: Uncertainty-Guided Work Allocation between Human and Large Language Models for Data Annotation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.15638","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:51a3f4b02e05dcbd3c2a30240f3618de083257b8f9701c04d1cd60abdf3b581c","target":"record","created_at":"2026-07-05T07:56:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5e56ea8152bcfb4ec9fb0d3ed700cbd0acd507ddccf5e67c234ccf8cafc924f0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-24T08:56:49Z","title_canon_sha256":"0e9e22e1e1019cbf27720a22238591ac0f799ae5bc86571b0ab4a33ad25926fb"},"schema_version":"1.0","source":{"id":"2310.15638","kind":"arxiv","version":1}},"canonical_sha256":"4915659820f81835c5f141e9db6febb0470684b9f4a8cbd8b41a89a50dd341cf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4915659820f81835c5f141e9db6febb0470684b9f4a8cbd8b41a89a50dd341cf","first_computed_at":"2026-07-05T07:56:18.163699Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:56:18.163699Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"T8KolUk47LpEVNOBNEkGnwWzjmSkm65A6eJuuxarb/jNwpO5HQMa5R0DnlWCsHFM+a5/eAZE7AbALfnyEDdZCg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:56:18.164202Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.15638","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:51a3f4b02e05dcbd3c2a30240f3618de083257b8f9701c04d1cd60abdf3b581c","sha256:38d35515bf7b58b0c1b12e5f393b1da29e9734a233ed4118d51397249ec71d17"],"state_sha256":"34c90689f0d66ca130589322afad9657565ffcf62d084e42b4144549e2967c5e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XFEHb24KnJJiy3qm4kicBixoKDy9kRdTZy7DtgvzuOeX6nDVUZMZZarmR7ZQgnZ0RT9wTuDUhp+J7+DUlv7HAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T09:37:59.068598Z","bundle_sha256":"3d626f2d862bf3f65adcbc14aec1815b0b75e8d12bbd96a0cc0f5c9da432e0ec"}}