{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:DUQDL3WTVWKITKRJYHL2OBTOQT","short_pith_number":"pith:DUQDL3WT","canonical_record":{"source":{"id":"2101.04615","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-01-12T17:14:18Z","cross_cats_sorted":["cs.IR","cs.LG"],"title_canon_sha256":"b7a41b1385de4f32efdafb044db0bbb7089802e99cf473abb5b36f5ebf35dd6a","abstract_canon_sha256":"2bf986a5563eacceb77b6c6c2475d1578d453fd1a02bd2de1d031b1d67e70131"},"schema_version":"1.0"},"canonical_sha256":"1d2035eed3ad9489aa29c1d7a7066e84e8a22fa7e68754b0f67697d4e38d71e8","source":{"kind":"arxiv","id":"2101.04615","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2101.04615","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"arxiv_version","alias_value":"2101.04615v2","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2101.04615","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_12","alias_value":"DUQDL3WTVWKI","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_16","alias_value":"DUQDL3WTVWKITKRJ","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_8","alias_value":"DUQDL3WT","created_at":"2026-07-05T02:28:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:DUQDL3WTVWKITKRJYHL2OBTOQT","target":"record","payload":{"canonical_record":{"source":{"id":"2101.04615","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-01-12T17:14:18Z","cross_cats_sorted":["cs.IR","cs.LG"],"title_canon_sha256":"b7a41b1385de4f32efdafb044db0bbb7089802e99cf473abb5b36f5ebf35dd6a","abstract_canon_sha256":"2bf986a5563eacceb77b6c6c2475d1578d453fd1a02bd2de1d031b1d67e70131"},"schema_version":"1.0"},"canonical_sha256":"1d2035eed3ad9489aa29c1d7a7066e84e8a22fa7e68754b0f67697d4e38d71e8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:28:55.092911Z","signature_b64":"qTAr6yY5NobvmhSNhzyvNnYZF7DbNH+sZqIvQSNazTbbnWc4qhnzm2yCgJrQ4ETFaATVZnSklpJDuCpc2Y8qAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1d2035eed3ad9489aa29c1d7a7066e84e8a22fa7e68754b0f67697d4e38d71e8","last_reissued_at":"2026-07-05T02:28:55.092492Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:28:55.092492Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2101.04615","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:28:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AhCRQ9bv0JWOCumURD048SloaGhUDHh7jVaqT3fCl/ZSQGMACDdXdSMG59lxmDHPhTKqxOUjZopoMRc0FklUBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T01:28:38.582161Z"},"content_sha256":"ea48d6c12952df1972f33fe84ae8af66f4610ca89f4449efe2c914fbdddc2255","schema_version":"1.0","event_id":"sha256:ea48d6c12952df1972f33fe84ae8af66f4610ca89f4449efe2c914fbdddc2255"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:DUQDL3WTVWKITKRJYHL2OBTOQT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Toward Effective Automated Content Analysis via Crowdsourcing","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.IR","cs.LG"],"primary_cat":"cs.CL","authors_text":"Chau-Wai Wong, Jiele Wu, Xianpeng Liu, Xinyan Zhao","submitted_at":"2021-01-12T17:14:18Z","abstract_excerpt":"Many computer scientists use the aggregated answers of online workers to represent ground truth. Prior work has shown that aggregation methods such as majority voting are effective for measuring relatively objective features. For subjective features such as semantic connotation, online workers, known for optimizing their hourly earnings, tend to deteriorate in the quality of their responses as they work longer. In this paper, we aim to address this issue by proposing a quality-aware semantic data annotation system. We observe that with timely feedback on workers' performance quantified by qual"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2101.04615","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2101.04615/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:28:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rIRYXJo5wLwZySz054Q835pWmV3kiTao58iPOHnYvHv5bAW3USzciMJWO8lLfLSqKuGCbpBeBqUxM6pheFbWBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T01:28:38.582557Z"},"content_sha256":"e2f39ba00af45d72e6e8c0dc51640dbecfd71c89a29c254fdf9ed5150a780682","schema_version":"1.0","event_id":"sha256:e2f39ba00af45d72e6e8c0dc51640dbecfd71c89a29c254fdf9ed5150a780682"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/bundle.json","state_url":"https://pith.science/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T01:28:38Z","links":{"resolver":"https://pith.science/pith/DUQDL3WTVWKITKRJYHL2OBTOQT","bundle":"https://pith.science/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/bundle.json","state":"https://pith.science/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DUQDL3WTVWKITKRJYHL2OBTOQT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:DUQDL3WTVWKITKRJYHL2OBTOQT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2bf986a5563eacceb77b6c6c2475d1578d453fd1a02bd2de1d031b1d67e70131","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-01-12T17:14:18Z","title_canon_sha256":"b7a41b1385de4f32efdafb044db0bbb7089802e99cf473abb5b36f5ebf35dd6a"},"schema_version":"1.0","source":{"id":"2101.04615","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2101.04615","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"arxiv_version","alias_value":"2101.04615v2","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2101.04615","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_12","alias_value":"DUQDL3WTVWKI","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_16","alias_value":"DUQDL3WTVWKITKRJ","created_at":"2026-07-05T02:28:55Z"},{"alias_kind":"pith_short_8","alias_value":"DUQDL3WT","created_at":"2026-07-05T02:28:55Z"}],"graph_snapshots":[{"event_id":"sha256:e2f39ba00af45d72e6e8c0dc51640dbecfd71c89a29c254fdf9ed5150a780682","target":"graph","created_at":"2026-07-05T02:28:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2101.04615/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Many computer scientists use the aggregated answers of online workers to represent ground truth. Prior work has shown that aggregation methods such as majority voting are effective for measuring relatively objective features. For subjective features such as semantic connotation, online workers, known for optimizing their hourly earnings, tend to deteriorate in the quality of their responses as they work longer. In this paper, we aim to address this issue by proposing a quality-aware semantic data annotation system. We observe that with timely feedback on workers' performance quantified by qual","authors_text":"Chau-Wai Wong, Jiele Wu, Xianpeng Liu, Xinyan Zhao","cross_cats":["cs.IR","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-01-12T17:14:18Z","title":"Toward Effective Automated Content Analysis via Crowdsourcing"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2101.04615","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ea48d6c12952df1972f33fe84ae8af66f4610ca89f4449efe2c914fbdddc2255","target":"record","created_at":"2026-07-05T02:28:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2bf986a5563eacceb77b6c6c2475d1578d453fd1a02bd2de1d031b1d67e70131","cross_cats_sorted":["cs.IR","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-01-12T17:14:18Z","title_canon_sha256":"b7a41b1385de4f32efdafb044db0bbb7089802e99cf473abb5b36f5ebf35dd6a"},"schema_version":"1.0","source":{"id":"2101.04615","kind":"arxiv","version":2}},"canonical_sha256":"1d2035eed3ad9489aa29c1d7a7066e84e8a22fa7e68754b0f67697d4e38d71e8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1d2035eed3ad9489aa29c1d7a7066e84e8a22fa7e68754b0f67697d4e38d71e8","first_computed_at":"2026-07-05T02:28:55.092492Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:28:55.092492Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qTAr6yY5NobvmhSNhzyvNnYZF7DbNH+sZqIvQSNazTbbnWc4qhnzm2yCgJrQ4ETFaATVZnSklpJDuCpc2Y8qAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T02:28:55.092911Z","signed_message":"canonical_sha256_bytes"},"source_id":"2101.04615","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ea48d6c12952df1972f33fe84ae8af66f4610ca89f4449efe2c914fbdddc2255","sha256:e2f39ba00af45d72e6e8c0dc51640dbecfd71c89a29c254fdf9ed5150a780682"],"state_sha256":"017349d6da02105bcc0246adabbf936e24c05a9647f04b56b39157038a01ab40"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+uHDXQm/yK5D1uR4ZOgxwg+sN876wuJoMlm7m9sNbvr17isfgrZn9VPIv2VYuNZksIHxAm3TEDatu18/q2wSBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T01:28:38.585103Z","bundle_sha256":"dbe8df751ec47d7c266ebe42000d686190cc0a8ae68b8602adecd0317a41d737"}}