{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2013:MIK7RSUREULTLGANDWMTPU6JMJ","short_pith_number":"pith:MIK7RSUR","schema_version":"1.0","canonical_sha256":"6215f8ca91251735980d1d9937d3c962545b4ec85d56addf673bff62bffe28c6","source":{"kind":"arxiv","id":"1305.0015","version":1},"attestation_state":"computed","paper":{"title":"Inferring ground truth from multi-annotator ordinal data: a probabilistic approach","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"stat.ML","authors_text":"Balaji Lakshminarayanan, Yee Whye Teh","submitted_at":"2013-04-30T20:12:01Z","abstract_excerpt":"A popular approach for large scale data annotation tasks is crowdsourcing, wherein each data point is labeled by multiple noisy annotators. We consider the problem of inferring ground truth from noisy ordinal labels obtained from multiple annotators of varying and unknown expertise levels. Annotation models for ordinal data have been proposed mostly as extensions of their binary/categorical counterparts and have received little attention in the crowdsourcing literature. We propose a new model for crowdsourced ordinal data that accounts for instance difficulty as well as annotator expertise, an"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1305.0015","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2013-04-30T20:12:01Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"6f96b95db3182721166d087a060a4d58b82e6136d8cb41875bd82501405d688e","abstract_canon_sha256":"e6ff3b6ed8b25f0f9e46558d96328dde86b1a6312af8f7bae6a719cb792c8053"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T03:26:42.274298Z","signature_b64":"qS+shDq4O7hFqUSoVFCbYUqruqqtirxnNlDOqzTKheSQUxHD89sW7T0AAVjM6YsuHg7EgnmSoZOekYA/5l7UCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6215f8ca91251735980d1d9937d3c962545b4ec85d56addf673bff62bffe28c6","last_reissued_at":"2026-05-18T03:26:42.273815Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T03:26:42.273815Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Inferring ground truth from multi-annotator ordinal data: a probabilistic approach","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"stat.ML","authors_text":"Balaji Lakshminarayanan, Yee Whye Teh","submitted_at":"2013-04-30T20:12:01Z","abstract_excerpt":"A popular approach for large scale data annotation tasks is crowdsourcing, wherein each data point is labeled by multiple noisy annotators. We consider the problem of inferring ground truth from noisy ordinal labels obtained from multiple annotators of varying and unknown expertise levels. Annotation models for ordinal data have been proposed mostly as extensions of their binary/categorical counterparts and have received little attention in the crowdsourcing literature. We propose a new model for crowdsourced ordinal data that accounts for instance difficulty as well as annotator expertise, an"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1305.0015","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1305.0015","created_at":"2026-05-18T03:26:42.273889+00:00"},{"alias_kind":"arxiv_version","alias_value":"1305.0015v1","created_at":"2026-05-18T03:26:42.273889+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1305.0015","created_at":"2026-05-18T03:26:42.273889+00:00"},{"alias_kind":"pith_short_12","alias_value":"MIK7RSUREULT","created_at":"2026-05-18T12:27:52.871228+00:00"},{"alias_kind":"pith_short_16","alias_value":"MIK7RSUREULTLGAN","created_at":"2026-05-18T12:27:52.871228+00:00"},{"alias_kind":"pith_short_8","alias_value":"MIK7RSUR","created_at":"2026-05-18T12:27:52.871228+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2507.12041","citing_title":"Granular feedback merits sophisticated aggregation","ref_index":2023,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ","json":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ.json","graph_json":"https://pith.science/api/pith-number/MIK7RSUREULTLGANDWMTPU6JMJ/graph.json","events_json":"https://pith.science/api/pith-number/MIK7RSUREULTLGANDWMTPU6JMJ/events.json","paper":"https://pith.science/paper/MIK7RSUR"},"agent_actions":{"view_html":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ","download_json":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ.json","view_paper":"https://pith.science/paper/MIK7RSUR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1305.0015&json=true","fetch_graph":"https://pith.science/api/pith-number/MIK7RSUREULTLGANDWMTPU6JMJ/graph.json","fetch_events":"https://pith.science/api/pith-number/MIK7RSUREULTLGANDWMTPU6JMJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ/action/storage_attestation","attest_author":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ/action/author_attestation","sign_citation":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ/action/citation_signature","submit_replication":"https://pith.science/pith/MIK7RSUREULTLGANDWMTPU6JMJ/action/replication_record"}},"created_at":"2026-05-18T03:26:42.273889+00:00","updated_at":"2026-05-18T03:26:42.273889+00:00"}