{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2018:OQO6HGD5CGZHU5RRJ2NRQZD2WX","short_pith_number":"pith:OQO6HGD5","schema_version":"1.0","canonical_sha256":"741de3987d11b27a76314e9b18647ab5d744aede17f1a6ebcd5a2e8241967252","source":{"kind":"arxiv","id":"1806.05004","version":1},"attestation_state":"computed","paper":{"title":"Explainable Agreement through Simulation for Tasks with Subjective Labels","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"John Foley","submitted_at":"2018-06-13T13:04:01Z","abstract_excerpt":"The field of information retrieval often works with limited and noisy data in an attempt to classify documents into subjective categories, e.g., relevance, sentiment and controversy. We typically quantify a notion of agreement to understand the difficulty of the labeling task, but when we present final results, we do so using measures that are unaware of agreement or the inherent subjectivity of the task. We propose using user simulation to understand the effect size of this noisy agreement data. By simulating truth and predictions, we can understand the maximum scores a dataset can support: f"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1806.05004","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2018-06-13T13:04:01Z","cross_cats_sorted":[],"title_canon_sha256":"7be7a338c20c8b876c4641e58519ae83834472403cd166c20b32dec6b72e39c3","abstract_canon_sha256":"36024064a1071c6f9ed958b79eca766a31b02ecd572df8484b18fb0870dfd373"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:13:20.374629Z","signature_b64":"nyVRYiajzgj0supe7pDk5xz6HSdWoSo/dVIyFuqcNtxhieZLMc5RaZb1PvLkAFE4qP4bjAJhjDD+qR3Gfva6CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"741de3987d11b27a76314e9b18647ab5d744aede17f1a6ebcd5a2e8241967252","last_reissued_at":"2026-05-18T00:13:20.374035Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:13:20.374035Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Explainable Agreement through Simulation for Tasks with Subjective Labels","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"John Foley","submitted_at":"2018-06-13T13:04:01Z","abstract_excerpt":"The field of information retrieval often works with limited and noisy data in an attempt to classify documents into subjective categories, e.g., relevance, sentiment and controversy. We typically quantify a notion of agreement to understand the difficulty of the labeling task, but when we present final results, we do so using measures that are unaware of agreement or the inherent subjectivity of the task. We propose using user simulation to understand the effect size of this noisy agreement data. By simulating truth and predictions, we can understand the maximum scores a dataset can support: f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1806.05004","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1806.05004","created_at":"2026-05-18T00:13:20.374127+00:00"},{"alias_kind":"arxiv_version","alias_value":"1806.05004v1","created_at":"2026-05-18T00:13:20.374127+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1806.05004","created_at":"2026-05-18T00:13:20.374127+00:00"},{"alias_kind":"pith_short_12","alias_value":"OQO6HGD5CGZH","created_at":"2026-05-18T12:32:43.782077+00:00"},{"alias_kind":"pith_short_16","alias_value":"OQO6HGD5CGZHU5RR","created_at":"2026-05-18T12:32:43.782077+00:00"},{"alias_kind":"pith_short_8","alias_value":"OQO6HGD5","created_at":"2026-05-18T12:32:43.782077+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX","json":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX.json","graph_json":"https://pith.science/api/pith-number/OQO6HGD5CGZHU5RRJ2NRQZD2WX/graph.json","events_json":"https://pith.science/api/pith-number/OQO6HGD5CGZHU5RRJ2NRQZD2WX/events.json","paper":"https://pith.science/paper/OQO6HGD5"},"agent_actions":{"view_html":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX","download_json":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX.json","view_paper":"https://pith.science/paper/OQO6HGD5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1806.05004&json=true","fetch_graph":"https://pith.science/api/pith-number/OQO6HGD5CGZHU5RRJ2NRQZD2WX/graph.json","fetch_events":"https://pith.science/api/pith-number/OQO6HGD5CGZHU5RRJ2NRQZD2WX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX/action/storage_attestation","attest_author":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX/action/author_attestation","sign_citation":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX/action/citation_signature","submit_replication":"https://pith.science/pith/OQO6HGD5CGZHU5RRJ2NRQZD2WX/action/replication_record"}},"created_at":"2026-05-18T00:13:20.374127+00:00","updated_at":"2026-05-18T00:13:20.374127+00:00"}