{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:A6PL2VSHYCY54YDGJ2TSUXJSVV","short_pith_number":"pith:A6PL2VSH","canonical_record":{"source":{"id":"1701.02185","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"2633b7bb72fcc0d3d72eca25a5b642aa6a2e827f923539ec2f5177f06a512988","abstract_canon_sha256":"7b44ce074c2104cba4a6c81aae90ae802ce16142df460b10fc9377ac0991addd"},"schema_version":"1.0"},"canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","source":{"kind":"arxiv","id":"1701.02185","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1701.02185","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"arxiv_version","alias_value":"1701.02185v2","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1701.02185","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"pith_short_12","alias_value":"A6PL2VSHYCY5","created_at":"2026-05-18T12:31:05Z"},{"alias_kind":"pith_short_16","alias_value":"A6PL2VSHYCY54YDG","created_at":"2026-05-18T12:31:05Z"},{"alias_kind":"pith_short_8","alias_value":"A6PL2VSH","created_at":"2026-05-18T12:31:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:A6PL2VSHYCY54YDGJ2TSUXJSVV","target":"record","payload":{"canonical_record":{"source":{"id":"1701.02185","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"2633b7bb72fcc0d3d72eca25a5b642aa6a2e827f923539ec2f5177f06a512988","abstract_canon_sha256":"7b44ce074c2104cba4a6c81aae90ae802ce16142df460b10fc9377ac0991addd"},"schema_version":"1.0"},"canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:04:47.642699Z","signature_b64":"cHd091MNoz+fIo9Sm8wUSMolmt43cwURtEWvTZeSQ6ywqTugMdUjsmyP9BGGUikh1IX9iGSO42m8qb98/Xg1Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","last_reissued_at":"2026-05-18T00:04:47.641916Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:04:47.641916Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1701.02185","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:04:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G2iupXB4IOC0s9Zu+qo67s26kLs8HFjOhbtk4V9Q1KxYhHkOv+QeCrCNVwLDjL/QboRyb4MlZd3WQPw4xuYWAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-11T14:08:28.070205Z"},"content_sha256":"b71e36ee9a4f6c5452c2b1343bb699056ebff09483fcea9e1a79b36d2f93a270","schema_version":"1.0","event_id":"sha256:b71e36ee9a4f6c5452c2b1343bb699056ebff09483fcea9e1a79b36d2f93a270"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:A6PL2VSHYCY54YDGJ2TSUXJSVV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Crowdsourcing Ground Truth for Medical Relation Extraction","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.CL","authors_text":"Anca Dumitrache, Chris Welty, Lora Aroyo","submitted_at":"2017-01-09T14:13:23Z","abstract_excerpt":"Cognitive computing systems require human labeled data for evaluation, and often for training. The standard practice used in gathering this data minimizes disagreement between annotators, and we have found this results in data that fails to account for the ambiguity inherent in language. We have proposed the CrowdTruth method for collecting ground truth through crowdsourcing, that reconsiders the role of people in machine learning based on the observation that disagreement between annotators provides a useful signal for phenomena such as ambiguity in the text. We report on using this method to"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1701.02185","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:04:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ch7RYz+4OY8ToIx4tY3Z5/FNnRhDZwa/T1u354BLXqqeGti32DJH46mX6MGDb0TrVmtLlFsbIr4jiup90Pa/CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-11T14:08:28.070556Z"},"content_sha256":"0cb7dc20134318be279ae957975181b4a71482b89ed5608dc8e2ceaed323151f","schema_version":"1.0","event_id":"sha256:0cb7dc20134318be279ae957975181b4a71482b89ed5608dc8e2ceaed323151f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/bundle.json","state_url":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-11T14:08:28Z","links":{"resolver":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV","bundle":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/bundle.json","state":"https://pith.science/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/A6PL2VSHYCY54YDGJ2TSUXJSVV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:A6PL2VSHYCY54YDGJ2TSUXJSVV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7b44ce074c2104cba4a6c81aae90ae802ce16142df460b10fc9377ac0991addd","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","title_canon_sha256":"2633b7bb72fcc0d3d72eca25a5b642aa6a2e827f923539ec2f5177f06a512988"},"schema_version":"1.0","source":{"id":"1701.02185","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1701.02185","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"arxiv_version","alias_value":"1701.02185v2","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1701.02185","created_at":"2026-05-18T00:04:47Z"},{"alias_kind":"pith_short_12","alias_value":"A6PL2VSHYCY5","created_at":"2026-05-18T12:31:05Z"},{"alias_kind":"pith_short_16","alias_value":"A6PL2VSHYCY54YDG","created_at":"2026-05-18T12:31:05Z"},{"alias_kind":"pith_short_8","alias_value":"A6PL2VSH","created_at":"2026-05-18T12:31:05Z"}],"graph_snapshots":[{"event_id":"sha256:0cb7dc20134318be279ae957975181b4a71482b89ed5608dc8e2ceaed323151f","target":"graph","created_at":"2026-05-18T00:04:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Cognitive computing systems require human labeled data for evaluation, and often for training. The standard practice used in gathering this data minimizes disagreement between annotators, and we have found this results in data that fails to account for the ambiguity inherent in language. We have proposed the CrowdTruth method for collecting ground truth through crowdsourcing, that reconsiders the role of people in machine learning based on the observation that disagreement between annotators provides a useful signal for phenomena such as ambiguity in the text. We report on using this method to","authors_text":"Anca Dumitrache, Chris Welty, Lora Aroyo","cross_cats":["cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","title":"Crowdsourcing Ground Truth for Medical Relation Extraction"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1701.02185","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b71e36ee9a4f6c5452c2b1343bb699056ebff09483fcea9e1a79b36d2f93a270","target":"record","created_at":"2026-05-18T00:04:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7b44ce074c2104cba4a6c81aae90ae802ce16142df460b10fc9377ac0991addd","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2017-01-09T14:13:23Z","title_canon_sha256":"2633b7bb72fcc0d3d72eca25a5b642aa6a2e827f923539ec2f5177f06a512988"},"schema_version":"1.0","source":{"id":"1701.02185","kind":"arxiv","version":2}},"canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"079ebd5647c0b1de60664ea72a5d32ad7df8fe61b0339f2196464232b7156a29","first_computed_at":"2026-05-18T00:04:47.641916Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:04:47.641916Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cHd091MNoz+fIo9Sm8wUSMolmt43cwURtEWvTZeSQ6ywqTugMdUjsmyP9BGGUikh1IX9iGSO42m8qb98/Xg1Ag==","signature_status":"signed_v1","signed_at":"2026-05-18T00:04:47.642699Z","signed_message":"canonical_sha256_bytes"},"source_id":"1701.02185","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b71e36ee9a4f6c5452c2b1343bb699056ebff09483fcea9e1a79b36d2f93a270","sha256:0cb7dc20134318be279ae957975181b4a71482b89ed5608dc8e2ceaed323151f"],"state_sha256":"f0f4d0bc4037679610e98cca8ad6c9ff1ae6a769e8adf649af8f80b60f189ab8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JEcVuOFktvhFige4zwq7cx0b4P2GQs+06x04HlcMM6unqh68yRLqHBHb9nTsJykxjqcyYBicHlsziTyPGWrOBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-11T14:08:28.072483Z","bundle_sha256":"d5aa6ee315c69cf70ef81b78e8c6a64948f8d4094eb67615db89af74f9dd8a9e"}}