{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:ORHOJELH24ZFPEFQGKKF4FDEM7","short_pith_number":"pith:ORHOJELH","canonical_record":{"source":{"id":"2110.05719","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T03:12:34Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"06e618ffeebc7878dfdc7d81f04402141e9e034df903e4581918918756be8021","abstract_canon_sha256":"d8b4f5e709da00c3afe73399d927098361adb5e706ce13aa817e38f25dd00dec"},"schema_version":"1.0"},"canonical_sha256":"744ee49167d7325790b032945e146467e70096884ab28ef9b17027fdd02b7095","source":{"kind":"arxiv","id":"2110.05719","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.05719","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"arxiv_version","alias_value":"2110.05719v1","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.05719","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_12","alias_value":"ORHOJELH24ZF","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_16","alias_value":"ORHOJELH24ZFPEFQ","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_8","alias_value":"ORHOJELH","created_at":"2026-07-05T03:22:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:ORHOJELH24ZFPEFQGKKF4FDEM7","target":"record","payload":{"canonical_record":{"source":{"id":"2110.05719","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T03:12:34Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"06e618ffeebc7878dfdc7d81f04402141e9e034df903e4581918918756be8021","abstract_canon_sha256":"d8b4f5e709da00c3afe73399d927098361adb5e706ce13aa817e38f25dd00dec"},"schema_version":"1.0"},"canonical_sha256":"744ee49167d7325790b032945e146467e70096884ab28ef9b17027fdd02b7095","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:22:05.725408Z","signature_b64":"sNHmIHr9rv6s4oT4/ITVr6nqlBaT7RTXns4jqgAMQZJ9UxoggiXfiM635U2sVgQh5k+FbvT3mnNlgUMLRfGoBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"744ee49167d7325790b032945e146467e70096884ab28ef9b17027fdd02b7095","last_reissued_at":"2026-07-05T03:22:05.724918Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:22:05.724918Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2110.05719","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:22:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qDgA7bx6Y1SZf/CVicRKoaTi/5QN95HhovNqug5mMt/mOt21BqwTzaKEQAz3S+GloGDxLxLZr+wcJiYZok7PAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T22:57:52.564466Z"},"content_sha256":"5dd0488e8bd52c98649699ad7b841fa4ec0912c7c66a07de25d1a2daa0aadce6","schema_version":"1.0","event_id":"sha256:5dd0488e8bd52c98649699ad7b841fa4ec0912c7c66a07de25d1a2daa0aadce6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:ORHOJELH24ZFPEFQGKKF4FDEM7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Dealing with Disagreements: Looking Beyond the Majority Vote in Subjective Annotations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.CL","authors_text":"Aida Mostafazadeh Davani, Mark D\\'iaz, Vinodkumar Prabhakaran","submitted_at":"2021-10-12T03:12:34Z","abstract_excerpt":"Majority voting and averaging are common approaches employed to resolve annotator disagreements and derive single ground truth labels from multiple annotations. However, annotators may systematically disagree with one another, often reflecting their individual biases and values, especially in the case of subjective tasks such as detecting affect, aggression, and hate speech. Annotator disagreements may capture important nuances in such tasks that are often ignored while aggregating annotations to a single ground truth. In order to address this, we investigate the efficacy of multi-annotator mo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.05719","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2110.05719/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:22:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"h7DofSCUsctUuE3e8ncrhQna8l7nGC1dMesyUl+agCZ/nFV6iACA+qochGqZZi2Ne7sUIBe8vBBDykXhHE6yBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T22:57:52.564975Z"},"content_sha256":"bb7ba7c84d75254cac51b7aecda71c8ae5861599d340d2a6d1784bfa38f9991f","schema_version":"1.0","event_id":"sha256:bb7ba7c84d75254cac51b7aecda71c8ae5861599d340d2a6d1784bfa38f9991f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/bundle.json","state_url":"https://pith.science/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T22:57:52Z","links":{"resolver":"https://pith.science/pith/ORHOJELH24ZFPEFQGKKF4FDEM7","bundle":"https://pith.science/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/bundle.json","state":"https://pith.science/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ORHOJELH24ZFPEFQGKKF4FDEM7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:ORHOJELH24ZFPEFQGKKF4FDEM7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d8b4f5e709da00c3afe73399d927098361adb5e706ce13aa817e38f25dd00dec","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T03:12:34Z","title_canon_sha256":"06e618ffeebc7878dfdc7d81f04402141e9e034df903e4581918918756be8021"},"schema_version":"1.0","source":{"id":"2110.05719","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.05719","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"arxiv_version","alias_value":"2110.05719v1","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.05719","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_12","alias_value":"ORHOJELH24ZF","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_16","alias_value":"ORHOJELH24ZFPEFQ","created_at":"2026-07-05T03:22:05Z"},{"alias_kind":"pith_short_8","alias_value":"ORHOJELH","created_at":"2026-07-05T03:22:05Z"}],"graph_snapshots":[{"event_id":"sha256:bb7ba7c84d75254cac51b7aecda71c8ae5861599d340d2a6d1784bfa38f9991f","target":"graph","created_at":"2026-07-05T03:22:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2110.05719/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Majority voting and averaging are common approaches employed to resolve annotator disagreements and derive single ground truth labels from multiple annotations. However, annotators may systematically disagree with one another, often reflecting their individual biases and values, especially in the case of subjective tasks such as detecting affect, aggression, and hate speech. Annotator disagreements may capture important nuances in such tasks that are often ignored while aggregating annotations to a single ground truth. In order to address this, we investigate the efficacy of multi-annotator mo","authors_text":"Aida Mostafazadeh Davani, Mark D\\'iaz, Vinodkumar Prabhakaran","cross_cats":["cs.CY"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T03:12:34Z","title":"Dealing with Disagreements: Looking Beyond the Majority Vote in Subjective Annotations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.05719","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5dd0488e8bd52c98649699ad7b841fa4ec0912c7c66a07de25d1a2daa0aadce6","target":"record","created_at":"2026-07-05T03:22:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d8b4f5e709da00c3afe73399d927098361adb5e706ce13aa817e38f25dd00dec","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-12T03:12:34Z","title_canon_sha256":"06e618ffeebc7878dfdc7d81f04402141e9e034df903e4581918918756be8021"},"schema_version":"1.0","source":{"id":"2110.05719","kind":"arxiv","version":1}},"canonical_sha256":"744ee49167d7325790b032945e146467e70096884ab28ef9b17027fdd02b7095","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"744ee49167d7325790b032945e146467e70096884ab28ef9b17027fdd02b7095","first_computed_at":"2026-07-05T03:22:05.724918Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:22:05.724918Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"sNHmIHr9rv6s4oT4/ITVr6nqlBaT7RTXns4jqgAMQZJ9UxoggiXfiM635U2sVgQh5k+FbvT3mnNlgUMLRfGoBg==","signature_status":"signed_v1","signed_at":"2026-07-05T03:22:05.725408Z","signed_message":"canonical_sha256_bytes"},"source_id":"2110.05719","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5dd0488e8bd52c98649699ad7b841fa4ec0912c7c66a07de25d1a2daa0aadce6","sha256:bb7ba7c84d75254cac51b7aecda71c8ae5861599d340d2a6d1784bfa38f9991f"],"state_sha256":"12f05162c6e263639c19c0f99ecad8f60fa971d36e476aad1fb3790876095fe3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pFu5hCuyGcrgKfzu7tT7iMxb6hG0x05QU60rTKAzloJx+T8Vj5PGcBge3gy7HY4lR6b+DiQcfZeh7AdrmWlDBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T22:57:52.570058Z","bundle_sha256":"d4af56fe25fe07bae827289c40add4eb34125b26e025595990f475969682f49c"}}