{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:2JKQJEW4ZLJN3CIIJ62ZJCN5M4","short_pith_number":"pith:2JKQJEW4","schema_version":"1.0","canonical_sha256":"d2550492dccad2dd89084fb59489bd67398700b755ce53ff3fc1b9eb413b84f3","source":{"kind":"arxiv","id":"2012.00714","version":1},"attestation_state":"computed","paper":{"title":"Debiasing Evaluations That are Biased by Evaluations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.IT","cs.LG","math.IT"],"primary_cat":"stat.ML","authors_text":"Ivan Stelmakh, Jingyan Wang, Nihar B. Shah, Yuting Wei","submitted_at":"2020-12-01T18:20:43Z","abstract_excerpt":"It is common to evaluate a set of items by soliciting people to rate them. For example, universities ask students to rate the teaching quality of their instructors, and conference organizers ask authors of submissions to evaluate the quality of the reviews. However, in these applications, students often give a higher rating to a course if they receive higher grades in a course, and authors often give a higher rating to the reviews if their papers are accepted to the conference. In this work, we call these external factors the \"outcome\" experienced by people, and consider the problem of mitigat"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2012.00714","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2020-12-01T18:20:43Z","cross_cats_sorted":["cs.IT","cs.LG","math.IT"],"title_canon_sha256":"d4e75020e900392f63834713930a412edcc529d451e58b95fa4e1f4141696345","abstract_canon_sha256":"e50fe24209fa4fcebd53b8b7d17094712c75df003d0a0fc55cc5326239945ad8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:56:19.281331Z","signature_b64":"hVWMrd7GHm744G8x5H4bN2SmSdKephDg7gAhNGP1tZUsnka8fWzgla92RI18YLokJSh85gteGBSoB0n9DdtYBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d2550492dccad2dd89084fb59489bd67398700b755ce53ff3fc1b9eb413b84f3","last_reissued_at":"2026-07-05T01:56:19.280960Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:56:19.280960Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Debiasing Evaluations That are Biased by Evaluations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.IT","cs.LG","math.IT"],"primary_cat":"stat.ML","authors_text":"Ivan Stelmakh, Jingyan Wang, Nihar B. Shah, Yuting Wei","submitted_at":"2020-12-01T18:20:43Z","abstract_excerpt":"It is common to evaluate a set of items by soliciting people to rate them. For example, universities ask students to rate the teaching quality of their instructors, and conference organizers ask authors of submissions to evaluate the quality of the reviews. However, in these applications, students often give a higher rating to a course if they receive higher grades in a course, and authors often give a higher rating to the reviews if their papers are accepted to the conference. In this work, we call these external factors the \"outcome\" experienced by people, and consider the problem of mitigat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2012.00714","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2012.00714/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2012.00714","created_at":"2026-07-05T01:56:19.281034+00:00"},{"alias_kind":"arxiv_version","alias_value":"2012.00714v1","created_at":"2026-07-05T01:56:19.281034+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2012.00714","created_at":"2026-07-05T01:56:19.281034+00:00"},{"alias_kind":"pith_short_12","alias_value":"2JKQJEW4ZLJN","created_at":"2026-07-05T01:56:19.281034+00:00"},{"alias_kind":"pith_short_16","alias_value":"2JKQJEW4ZLJN3CII","created_at":"2026-07-05T01:56:19.281034+00:00"},{"alias_kind":"pith_short_8","alias_value":"2JKQJEW4","created_at":"2026-07-05T01:56:19.281034+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4","json":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4.json","graph_json":"https://pith.science/api/pith-number/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/graph.json","events_json":"https://pith.science/api/pith-number/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/events.json","paper":"https://pith.science/paper/2JKQJEW4"},"agent_actions":{"view_html":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4","download_json":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4.json","view_paper":"https://pith.science/paper/2JKQJEW4","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2012.00714&json=true","fetch_graph":"https://pith.science/api/pith-number/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/graph.json","fetch_events":"https://pith.science/api/pith-number/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/action/storage_attestation","attest_author":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/action/author_attestation","sign_citation":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/action/citation_signature","submit_replication":"https://pith.science/pith/2JKQJEW4ZLJN3CIIJ62ZJCN5M4/action/replication_record"}},"created_at":"2026-07-05T01:56:19.281034+00:00","updated_at":"2026-07-05T01:56:19.281034+00:00"}