{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:YWE32PQ424JMC752T4GBAQOFR3","short_pith_number":"pith:YWE32PQ4","canonical_record":{"source":{"id":"2212.09503","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T20:12:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"b84ebb2c72d020fdb29ad9b3edab189021a69b31b1112d9b961385a0351f92ec","abstract_canon_sha256":"c1ad85e49e496c918c2402655ef4c853e28736e34566e8f65079f7aedacecfd3"},"schema_version":"1.0"},"canonical_sha256":"c589bd3e1cd712c17fba9f0c1041c58ee3caf340e6559a46377b761bed6e772b","source":{"kind":"arxiv","id":"2212.09503","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.09503","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"arxiv_version","alias_value":"2212.09503v1","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.09503","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_12","alias_value":"YWE32PQ424JM","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_16","alias_value":"YWE32PQ424JMC752","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_8","alias_value":"YWE32PQ4","created_at":"2026-07-05T05:26:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:YWE32PQ424JMC752T4GBAQOFR3","target":"record","payload":{"canonical_record":{"source":{"id":"2212.09503","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T20:12:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"b84ebb2c72d020fdb29ad9b3edab189021a69b31b1112d9b961385a0351f92ec","abstract_canon_sha256":"c1ad85e49e496c918c2402655ef4c853e28736e34566e8f65079f7aedacecfd3"},"schema_version":"1.0"},"canonical_sha256":"c589bd3e1cd712c17fba9f0c1041c58ee3caf340e6559a46377b761bed6e772b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:26:22.931063Z","signature_b64":"gju+oGl/RGwj0lIKUjaEK4bElUhhH3hwhtM2FLd3EziEYghYPlx1FDFf/xchLaDyhctbVTeld42uQYNj7pwLBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c589bd3e1cd712c17fba9f0c1041c58ee3caf340e6559a46377b761bed6e772b","last_reissued_at":"2026-07-05T05:26:22.930577Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:26:22.930577Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2212.09503","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:26:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N25+1KOXf5DMpmT8+ortmvTafhiqAOwYzpErXmjBmtzwVBI3Y7WdUQ8W8lFDH51/9MXfmRkBl1OC0oVGq1o8Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T12:07:51.954071Z"},"content_sha256":"12eed0fe849d6459abe04bc4b9689bdaaee52dfce6fd19a90bc1818adac2b835","schema_version":"1.0","event_id":"sha256:12eed0fe849d6459abe04bc4b9689bdaaee52dfce6fd19a90bc1818adac2b835"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:YWE32PQ424JMC752T4GBAQOFR3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Measuring Annotator Agreement Generally across Complex Structured, Multi-object, and Free-text Annotation Tasks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alexander Braylan, Matthew Lease, Omar Alonso","submitted_at":"2022-12-15T20:12:48Z","abstract_excerpt":"When annotators label data, a key metric for quality assurance is inter-annotator agreement (IAA): the extent to which annotators agree on their labels. Though many IAA measures exist for simple categorical and ordinal labeling tasks, relatively little work has considered more complex labeling tasks, such as structured, multi-object, and free-text annotations. Krippendorff's alpha, best known for use with simpler labeling tasks, does have a distance-based formulation with broader applicability, but little work has studied its efficacy and consistency across complex annotation tasks.\n  We inves"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.09503","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.09503/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:26:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"R+WVBsNEWolC817LhhXFZ+hE9dcWj/p44iQpFXPLhJNvAwMJ7jElGpM2bd+cLwuLpXyuBePzZX1gRDusBZxPBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T12:07:51.955019Z"},"content_sha256":"9c51a435aae686f0266782b90b48042e32572357870ec228b55973e9dfe6c7f2","schema_version":"1.0","event_id":"sha256:9c51a435aae686f0266782b90b48042e32572357870ec228b55973e9dfe6c7f2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YWE32PQ424JMC752T4GBAQOFR3/bundle.json","state_url":"https://pith.science/pith/YWE32PQ424JMC752T4GBAQOFR3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YWE32PQ424JMC752T4GBAQOFR3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T12:07:51Z","links":{"resolver":"https://pith.science/pith/YWE32PQ424JMC752T4GBAQOFR3","bundle":"https://pith.science/pith/YWE32PQ424JMC752T4GBAQOFR3/bundle.json","state":"https://pith.science/pith/YWE32PQ424JMC752T4GBAQOFR3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YWE32PQ424JMC752T4GBAQOFR3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:YWE32PQ424JMC752T4GBAQOFR3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c1ad85e49e496c918c2402655ef4c853e28736e34566e8f65079f7aedacecfd3","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T20:12:48Z","title_canon_sha256":"b84ebb2c72d020fdb29ad9b3edab189021a69b31b1112d9b961385a0351f92ec"},"schema_version":"1.0","source":{"id":"2212.09503","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.09503","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"arxiv_version","alias_value":"2212.09503v1","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.09503","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_12","alias_value":"YWE32PQ424JM","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_16","alias_value":"YWE32PQ424JMC752","created_at":"2026-07-05T05:26:22Z"},{"alias_kind":"pith_short_8","alias_value":"YWE32PQ4","created_at":"2026-07-05T05:26:22Z"}],"graph_snapshots":[{"event_id":"sha256:9c51a435aae686f0266782b90b48042e32572357870ec228b55973e9dfe6c7f2","target":"graph","created_at":"2026-07-05T05:26:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2212.09503/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"When annotators label data, a key metric for quality assurance is inter-annotator agreement (IAA): the extent to which annotators agree on their labels. Though many IAA measures exist for simple categorical and ordinal labeling tasks, relatively little work has considered more complex labeling tasks, such as structured, multi-object, and free-text annotations. Krippendorff's alpha, best known for use with simpler labeling tasks, does have a distance-based formulation with broader applicability, but little work has studied its efficacy and consistency across complex annotation tasks.\n  We inves","authors_text":"Alexander Braylan, Matthew Lease, Omar Alonso","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T20:12:48Z","title":"Measuring Annotator Agreement Generally across Complex Structured, Multi-object, and Free-text Annotation Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.09503","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:12eed0fe849d6459abe04bc4b9689bdaaee52dfce6fd19a90bc1818adac2b835","target":"record","created_at":"2026-07-05T05:26:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c1ad85e49e496c918c2402655ef4c853e28736e34566e8f65079f7aedacecfd3","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T20:12:48Z","title_canon_sha256":"b84ebb2c72d020fdb29ad9b3edab189021a69b31b1112d9b961385a0351f92ec"},"schema_version":"1.0","source":{"id":"2212.09503","kind":"arxiv","version":1}},"canonical_sha256":"c589bd3e1cd712c17fba9f0c1041c58ee3caf340e6559a46377b761bed6e772b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c589bd3e1cd712c17fba9f0c1041c58ee3caf340e6559a46377b761bed6e772b","first_computed_at":"2026-07-05T05:26:22.930577Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:26:22.930577Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gju+oGl/RGwj0lIKUjaEK4bElUhhH3hwhtM2FLd3EziEYghYPlx1FDFf/xchLaDyhctbVTeld42uQYNj7pwLBw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:26:22.931063Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.09503","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:12eed0fe849d6459abe04bc4b9689bdaaee52dfce6fd19a90bc1818adac2b835","sha256:9c51a435aae686f0266782b90b48042e32572357870ec228b55973e9dfe6c7f2"],"state_sha256":"b1846d1a884934956490d45066165bb7572412c79eb6031da19ff13baf1d9e2b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UHJmX7/eGTm1Y1eW8VGvfN3F1DQW/f8tsIgui5hU9GD76KnvM6FnR93GptYZ49kWaGlqIDXJkpPqpDU/EvQLDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T12:07:51.961405Z","bundle_sha256":"256671e200eeb83ec93c439c1619ac19f029d77c6099d9fa8bc670ef95c65b84"}}