{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:FWOYROQGU7ULMP3GUP2VBZ72PL","short_pith_number":"pith:FWOYROQG","schema_version":"1.0","canonical_sha256":"2d9d88ba06a7e8b63f66a3f550e7fa7ac36b13b42abd830ae8fb1d4699ce5b3d","source":{"kind":"arxiv","id":"2106.05498","version":3},"attestation_state":"computed","paper":{"title":"It's COMPASlicated: The Messy Relationship between RAI Datasets and Algorithmic Fairness Benchmarks","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Aaron Horowitz, Angela Zhou, Brian Brubach, Kristian Lum, Michelle Bao, Samantha Zottola, Sarah Desmarais, Suresh Venkatasubramanian","submitted_at":"2021-06-10T04:59:06Z","abstract_excerpt":"Risk assessment instrument (RAI) datasets, particularly ProPublica's COMPAS dataset, are commonly used in algorithmic fairness papers due to benchmarking practices of comparing algorithms on datasets used in prior work. In many cases, this data is used as a benchmark to demonstrate good performance without accounting for the complexities of criminal justice (CJ) processes. However, we show that pretrial RAI datasets can contain numerous measurement biases and errors, and due to disparities in discretion and deployment, algorithmic fairness applied to RAI datasets is limited in making claims ab"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2106.05498","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CY","submitted_at":"2021-06-10T04:59:06Z","cross_cats_sorted":[],"title_canon_sha256":"75ffff0007011afc78b7bcb37970b376f9795305fde7dbc75a39eca4d4a1d35b","abstract_canon_sha256":"7e548720eff404ed003eadb6fe7303f49b0fcf8f00846125fd0480f2dc2b53a8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:18:47.024376Z","signature_b64":"49/86Ix5q92U+xm8jask6I3d4KXrJUFPT56QnT61xBD7mQ52N6C2I88s0hraSsLLrKiCBp5lExu4HNDEx8qRBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2d9d88ba06a7e8b63f66a3f550e7fa7ac36b13b42abd830ae8fb1d4699ce5b3d","last_reissued_at":"2026-07-05T04:18:47.023938Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:18:47.023938Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"It's COMPASlicated: The Messy Relationship between RAI Datasets and Algorithmic Fairness Benchmarks","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Aaron Horowitz, Angela Zhou, Brian Brubach, Kristian Lum, Michelle Bao, Samantha Zottola, Sarah Desmarais, Suresh Venkatasubramanian","submitted_at":"2021-06-10T04:59:06Z","abstract_excerpt":"Risk assessment instrument (RAI) datasets, particularly ProPublica's COMPAS dataset, are commonly used in algorithmic fairness papers due to benchmarking practices of comparing algorithms on datasets used in prior work. In many cases, this data is used as a benchmark to demonstrate good performance without accounting for the complexities of criminal justice (CJ) processes. However, we show that pretrial RAI datasets can contain numerous measurement biases and errors, and due to disparities in discretion and deployment, algorithmic fairness applied to RAI datasets is limited in making claims ab"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.05498","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.05498/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2106.05498","created_at":"2026-07-05T04:18:47.023996+00:00"},{"alias_kind":"arxiv_version","alias_value":"2106.05498v3","created_at":"2026-07-05T04:18:47.023996+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.05498","created_at":"2026-07-05T04:18:47.023996+00:00"},{"alias_kind":"pith_short_12","alias_value":"FWOYROQGU7UL","created_at":"2026-07-05T04:18:47.023996+00:00"},{"alias_kind":"pith_short_16","alias_value":"FWOYROQGU7ULMP3G","created_at":"2026-07-05T04:18:47.023996+00:00"},{"alias_kind":"pith_short_8","alias_value":"FWOYROQG","created_at":"2026-07-05T04:18:47.023996+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25668","citing_title":"Bridging Predictions and Interventions: An Integrated Framework for Automated Decision-Systems","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2606.02198","citing_title":"Model Multiplicity and Predictive Arbitrariness in Recidivism Risk Assessment","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2309.07176","citing_title":"Mind the Gap: Optimal and Equitable Encouragement Policies","ref_index":10,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL","json":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL.json","graph_json":"https://pith.science/api/pith-number/FWOYROQGU7ULMP3GUP2VBZ72PL/graph.json","events_json":"https://pith.science/api/pith-number/FWOYROQGU7ULMP3GUP2VBZ72PL/events.json","paper":"https://pith.science/paper/FWOYROQG"},"agent_actions":{"view_html":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL","download_json":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL.json","view_paper":"https://pith.science/paper/FWOYROQG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2106.05498&json=true","fetch_graph":"https://pith.science/api/pith-number/FWOYROQGU7ULMP3GUP2VBZ72PL/graph.json","fetch_events":"https://pith.science/api/pith-number/FWOYROQGU7ULMP3GUP2VBZ72PL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL/action/storage_attestation","attest_author":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL/action/author_attestation","sign_citation":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL/action/citation_signature","submit_replication":"https://pith.science/pith/FWOYROQGU7ULMP3GUP2VBZ72PL/action/replication_record"}},"created_at":"2026-07-05T04:18:47.023996+00:00","updated_at":"2026-07-05T04:18:47.023996+00:00"}