{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:YJNJBJMDPBERQIRDKNFWNPWNS5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"50cf340df67b5ea117bcc27a3c623ab4d3828296834cceaf250617a3055f53e4","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T02:35:45Z","title_canon_sha256":"8bbed539a82f0c3bb0d3a49824658f26ae22dc81b445571ac5f999735dfdfa63"},"schema_version":"1.0","source":{"id":"2110.05699","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.05699","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"arxiv_version","alias_value":"2110.05699v1","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.05699","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_12","alias_value":"YJNJBJMDPBER","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_16","alias_value":"YJNJBJMDPBERQIRD","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_8","alias_value":"YJNJBJMD","created_at":"2026-07-05T03:22:05Z"}],"graph_snapshots":[{"event_id":"sha256:9a4603e7fd64dbb74fba50e556e11687d59e4232ff60c8b8fd0503b3e120c695","target":"graph","created_at":"2026-07-05T03:22:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2110.05699/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"A common practice in building NLP datasets, especially using crowd-sourced annotations, involves obtaining multiple annotator judgements on the same data instances, which are then flattened to produce a single \"ground truth\" label or score, through majority voting, averaging, or adjudication. While these approaches may be appropriate in certain annotation tasks, such aggregations overlook the socially constructed nature of human perceptions that annotations for relatively more subjective tasks are meant to capture. In particular, systematic disagreements between annotators owing to their socio","authors_text":"Aida Mostafazadeh Davani, Mark D\\'iaz, Vinodkumar Prabhakaran","cross_cats":["cs.CY"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T02:35:45Z","title":"On Releasing Annotator-Level Labels and Information in Datasets"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.05699","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8975d55919ace1085da17d41bdd9d90306e5abe2e8f0e4b50029727f9ce86feb","target":"record","created_at":"2026-07-05T03:22:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"50cf340df67b5ea117bcc27a3c623ab4d3828296834cceaf250617a3055f53e4","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T02:35:45Z","title_canon_sha256":"8bbed539a82f0c3bb0d3a49824658f26ae22dc81b445571ac5f999735dfdfa63"},"schema_version":"1.0","source":{"id":"2110.05699","kind":"arxiv","version":1}},"canonical_sha256":"c25a90a5837849182223534b66becd9750580f4cdb4eae0839d60050e8f43c0f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c25a90a5837849182223534b66becd9750580f4cdb4eae0839d60050e8f43c0f","first_computed_at":"2026-07-05T03:22:05.454183Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:22:05.454183Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UeCyHR5YTGP17z4CRvx0lTMptPZCThbiVS3djJ6/kB0Blv2YZkpAErPKaI9qdJn9QLaqNKjl9DLe3+ZLVWOfCw==","signature_status":"signed_v1","signed_at":"2026-07-05T03:22:05.454675Z","signed_message":"canonical_sha256_bytes"},"source_id":"2110.05699","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8975d55919ace1085da17d41bdd9d90306e5abe2e8f0e4b50029727f9ce86feb","sha256:9a4603e7fd64dbb74fba50e556e11687d59e4232ff60c8b8fd0503b3e120c695"],"state_sha256":"dda30863b7fb97fd8ea1cc1fa2eddbe46ff0a32a3baf9248aef3b6f94f05e6b3"}