{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:27WQN5JPK7KOX7X2H6ONVTZAQW","short_pith_number":"pith:27WQN5JP","schema_version":"1.0","canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","source":{"kind":"arxiv","id":"2405.01719","version":2},"attestation_state":"computed","paper":{"title":"Inherent Trade-Offs between Diversity and Stability in Multi-Task Benchmarks","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Guanhua Zhang, Moritz Hardt","submitted_at":"2024-05-02T20:28:54Z","abstract_excerpt":"We examine multi-task benchmarks in machine learning through the lens of social choice theory. We draw an analogy between benchmarks and electoral systems, where models are candidates and tasks are voters. This suggests a distinction between cardinal and ordinal benchmark systems. The former aggregate numerical scores into one model ranking; the latter aggregate rankings for each task. We apply Arrow's impossibility theorem to ordinal benchmarks to highlight the inherent limitations of ordinal systems, particularly their sensitivity to the inclusion of irrelevant models. Inspired by Arrow's th"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.01719","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","cross_cats_sorted":[],"title_canon_sha256":"198fffc6747d285ef3be5740776b42fd6b975842d110bd8f146b9eaa3fa713a0","abstract_canon_sha256":"568939088ae2ab6603744f5d38c042a157825e4e9c4653dd4a7f6417b58be45d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:15:50.930538Z","signature_b64":"fYv3LpznR5g8ZotgcVEVmDFFYXgfO1YpXn18mIb2Vwa6xd4KHK0ZvYMW6783lN7QTyEGjtPXp2a9IoYqXJGDAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","last_reissued_at":"2026-07-05T08:15:50.929994Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:15:50.929994Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Inherent Trade-Offs between Diversity and Stability in Multi-Task Benchmarks","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Guanhua Zhang, Moritz Hardt","submitted_at":"2024-05-02T20:28:54Z","abstract_excerpt":"We examine multi-task benchmarks in machine learning through the lens of social choice theory. We draw an analogy between benchmarks and electoral systems, where models are candidates and tasks are voters. This suggests a distinction between cardinal and ordinal benchmark systems. The former aggregate numerical scores into one model ranking; the latter aggregate rankings for each task. We apply Arrow's impossibility theorem to ordinal benchmarks to highlight the inherent limitations of ordinal systems, particularly their sensitivity to the inclusion of irrelevant models. Inspired by Arrow's th"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01719","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.01719/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.01719","created_at":"2026-07-05T08:15:50.930053+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.01719v2","created_at":"2026-07-05T08:15:50.930053+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01719","created_at":"2026-07-05T08:15:50.930053+00:00"},{"alias_kind":"pith_short_12","alias_value":"27WQN5JPK7KO","created_at":"2026-07-05T08:15:50.930053+00:00"},{"alias_kind":"pith_short_16","alias_value":"27WQN5JPK7KOX7X2","created_at":"2026-07-05T08:15:50.930053+00:00"},{"alias_kind":"pith_short_8","alias_value":"27WQN5JP","created_at":"2026-07-05T08:15:50.930053+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.10967","citing_title":"Quo Vadis, Visual In-Context Learning? A Unified Benchmark Across Domains and Tasks","ref_index":108,"is_internal_anchor":false},{"citing_arxiv_id":"2504.16093","citing_title":"Efficient Portfolio Selection through Preference Aggregation with Quicksort and the Bradley--Terry Model","ref_index":54,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW","json":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW.json","graph_json":"https://pith.science/api/pith-number/27WQN5JPK7KOX7X2H6ONVTZAQW/graph.json","events_json":"https://pith.science/api/pith-number/27WQN5JPK7KOX7X2H6ONVTZAQW/events.json","paper":"https://pith.science/paper/27WQN5JP"},"agent_actions":{"view_html":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW","download_json":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW.json","view_paper":"https://pith.science/paper/27WQN5JP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.01719&json=true","fetch_graph":"https://pith.science/api/pith-number/27WQN5JPK7KOX7X2H6ONVTZAQW/graph.json","fetch_events":"https://pith.science/api/pith-number/27WQN5JPK7KOX7X2H6ONVTZAQW/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/action/timestamp_anchor","attest_storage":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/action/storage_attestation","attest_author":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/action/author_attestation","sign_citation":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/action/citation_signature","submit_replication":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/action/replication_record"}},"created_at":"2026-07-05T08:15:50.930053+00:00","updated_at":"2026-07-05T08:15:50.930053+00:00"}