{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2017:A6PL2VSHYCY54YDGJ2TSUXJSVV","short_pith_number":"pith:A6PL2VSH","schema_version":"1.0","canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","source":{"kind":"arxiv","id":"1701.02185","version":2},"attestation_state":"computed","paper":{"title":"Crowdsourcing Ground Truth for Medical Relation Extraction","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.CL","authors_text":"Anca Dumitrache, Chris Welty, Lora Aroyo","submitted_at":"2017-01-09T14:13:23Z","abstract_excerpt":"Cognitive computing systems require human labeled data for evaluation, and often for training. The standard practice used in gathering this data minimizes disagreement between annotators, and we have found this results in data that fails to account for the ambiguity inherent in language. We have proposed the CrowdTruth method for collecting ground truth through crowdsourcing, that reconsiders the role of people in machine learning based on the observation that disagreement between annotators provides a useful signal for phenomena such as ambiguity in the text. We report on using this method to"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1701.02185","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"2633b7bb72fcc0d3d72eca25a5b642aa6a2e827f923539ec2f5177f06a512988","abstract_canon_sha256":"7b44ce074c2104cba4a6c81aae90ae802ce16142df460b10fc9377ac0991addd"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:04:47.642699Z","signature_b64":"cHd091MNoz+fIo9Sm8wUSMolmt43cwURtEWvTZeSQ6ywqTugMdUjsmyP9BGGUikh1IX9iGSO42m8qb98/Xg1Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","last_reissued_at":"2026-05-18T00:04:47.641916Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:04:47.641916Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Crowdsourcing Ground Truth for Medical Relation Extraction","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.CL","authors_text":"Anca Dumitrache, Chris Welty, Lora Aroyo","submitted_at":"2017-01-09T14:13:23Z","abstract_excerpt":"Cognitive computing systems require human labeled data for evaluation, and often for training. The standard practice used in gathering this data minimizes disagreement between annotators, and we have found this results in data that fails to account for the ambiguity inherent in language. We have proposed the CrowdTruth method for collecting ground truth through crowdsourcing, that reconsiders the role of people in machine learning based on the observation that disagreement between annotators provides a useful signal for phenomena such as ambiguity in the text. We report on using this method to"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1701.02185","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1701.02185","created_at":"2026-05-18T00:04:47.642030+00:00"},{"alias_kind":"arxiv_version","alias_value":"1701.02185v2","created_at":"2026-05-18T00:04:47.642030+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1701.02185","created_at":"2026-05-18T00:04:47.642030+00:00"},{"alias_kind":"pith_short_12","alias_value":"A6PL2VSHYCY5","created_at":"2026-05-18T12:31:05.417338+00:00"},{"alias_kind":"pith_short_16","alias_value":"A6PL2VSHYCY54YDG","created_at":"2026-05-18T12:31:05.417338+00:00"},{"alias_kind":"pith_short_8","alias_value":"A6PL2VSH","created_at":"2026-05-18T12:31:05.417338+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV","json":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV.json","graph_json":"https://pith.science/api/pith-number/A6PL2VSHYCY54YDGJ2TSUXJSVV/graph.json","events_json":"https://pith.science/api/pith-number/A6PL2VSHYCY54YDGJ2TSUXJSVV/events.json","paper":"https://pith.science/paper/A6PL2VSH"},"agent_actions":{"view_html":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV","download_json":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV.json","view_paper":"https://pith.science/paper/A6PL2VSH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1701.02185&json=true","fetch_graph":"https://pith.science/api/pith-number/A6PL2VSHYCY54YDGJ2TSUXJSVV/graph.json","fetch_events":"https://pith.science/api/pith-number/A6PL2VSHYCY54YDGJ2TSUXJSVV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/action/storage_attestation","attest_author":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/action/author_attestation","sign_citation":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/action/citation_signature","submit_replication":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/action/replication_record"}},"created_at":"2026-05-18T00:04:47.642030+00:00","updated_at":"2026-05-18T00:04:47.642030+00:00"}