{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:27WQN5JPK7KOX7X2H6ONVTZAQW","short_pith_number":"pith:27WQN5JP","canonical_record":{"source":{"id":"2405.01719","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","cross_cats_sorted":[],"title_canon_sha256":"198fffc6747d285ef3be5740776b42fd6b975842d110bd8f146b9eaa3fa713a0","abstract_canon_sha256":"568939088ae2ab6603744f5d38c042a157825e4e9c4653dd4a7f6417b58be45d"},"schema_version":"1.0"},"canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","source":{"kind":"arxiv","id":"2405.01719","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.01719","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"arxiv_version","alias_value":"2405.01719v2","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01719","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_12","alias_value":"27WQN5JPK7KO","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_16","alias_value":"27WQN5JPK7KOX7X2","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_8","alias_value":"27WQN5JP","created_at":"2026-07-05T08:15:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:27WQN5JPK7KOX7X2H6ONVTZAQW","target":"record","payload":{"canonical_record":{"source":{"id":"2405.01719","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","cross_cats_sorted":[],"title_canon_sha256":"198fffc6747d285ef3be5740776b42fd6b975842d110bd8f146b9eaa3fa713a0","abstract_canon_sha256":"568939088ae2ab6603744f5d38c042a157825e4e9c4653dd4a7f6417b58be45d"},"schema_version":"1.0"},"canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:15:50.930538Z","signature_b64":"fYv3LpznR5g8ZotgcVEVmDFFYXgfO1YpXn18mIb2Vwa6xd4KHK0ZvYMW6783lN7QTyEGjtPXp2a9IoYqXJGDAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","last_reissued_at":"2026-07-05T08:15:50.929994Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:15:50.929994Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.01719","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:15:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k+l3klaWBtcz7PZvbw69pxaE/dGV9dEWAKiskYBjouQQsGLjQTCB52fKV4wPL8bSmbgzYfB8d7IVp4eWBqwOCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T05:15:33.812717Z"},"content_sha256":"c3fe7157c48cad06c852c58c9a4492e72fa24f2eeb4dc028eb573192d752c97e","schema_version":"1.0","event_id":"sha256:c3fe7157c48cad06c852c58c9a4492e72fa24f2eeb4dc028eb573192d752c97e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:27WQN5JPK7KOX7X2H6ONVTZAQW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Inherent Trade-Offs between Diversity and Stability in Multi-Task Benchmarks","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Guanhua Zhang, Moritz Hardt","submitted_at":"2024-05-02T20:28:54Z","abstract_excerpt":"We examine multi-task benchmarks in machine learning through the lens of social choice theory. We draw an analogy between benchmarks and electoral systems, where models are candidates and tasks are voters. This suggests a distinction between cardinal and ordinal benchmark systems. The former aggregate numerical scores into one model ranking; the latter aggregate rankings for each task. We apply Arrow's impossibility theorem to ordinal benchmarks to highlight the inherent limitations of ordinal systems, particularly their sensitivity to the inclusion of irrelevant models. Inspired by Arrow's th"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01719","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.01719/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:15:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mWDlt4ypNnadNadiRJFBRUIBYypaTrz3B5KrqzoiKz7Llt5oMaCUVQsi8D7UIePZ5VfsoUtufFhU6Qx9pMJ4BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T05:15:33.813074Z"},"content_sha256":"e5523c014ab81e427a26a797610bd3ffcc3ac754247d0c56709ebd16cbbc384c","schema_version":"1.0","event_id":"sha256:e5523c014ab81e427a26a797610bd3ffcc3ac754247d0c56709ebd16cbbc384c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/bundle.json","state_url":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T05:15:33Z","links":{"resolver":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW","bundle":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/bundle.json","state":"https://pith.science/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/27WQN5JPK7KOX7X2H6ONVTZAQW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:27WQN5JPK7KOX7X2H6ONVTZAQW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"568939088ae2ab6603744f5d38c042a157825e4e9c4653dd4a7f6417b58be45d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","title_canon_sha256":"198fffc6747d285ef3be5740776b42fd6b975842d110bd8f146b9eaa3fa713a0"},"schema_version":"1.0","source":{"id":"2405.01719","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.01719","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"arxiv_version","alias_value":"2405.01719v2","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.01719","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_12","alias_value":"27WQN5JPK7KO","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_16","alias_value":"27WQN5JPK7KOX7X2","created_at":"2026-07-05T08:15:50Z"},{"alias_kind":"pith_short_8","alias_value":"27WQN5JP","created_at":"2026-07-05T08:15:50Z"}],"graph_snapshots":[{"event_id":"sha256:e5523c014ab81e427a26a797610bd3ffcc3ac754247d0c56709ebd16cbbc384c","target":"graph","created_at":"2026-07-05T08:15:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.01719/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We examine multi-task benchmarks in machine learning through the lens of social choice theory. We draw an analogy between benchmarks and electoral systems, where models are candidates and tasks are voters. This suggests a distinction between cardinal and ordinal benchmark systems. The former aggregate numerical scores into one model ranking; the latter aggregate rankings for each task. We apply Arrow's impossibility theorem to ordinal benchmarks to highlight the inherent limitations of ordinal systems, particularly their sensitivity to the inclusion of irrelevant models. Inspired by Arrow's th","authors_text":"Guanhua Zhang, Moritz Hardt","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","title":"Inherent Trade-Offs between Diversity and Stability in Multi-Task Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.01719","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c3fe7157c48cad06c852c58c9a4492e72fa24f2eeb4dc028eb573192d752c97e","target":"record","created_at":"2026-07-05T08:15:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"568939088ae2ab6603744f5d38c042a157825e4e9c4653dd4a7f6417b58be45d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-02T20:28:54Z","title_canon_sha256":"198fffc6747d285ef3be5740776b42fd6b975842d110bd8f146b9eaa3fa713a0"},"schema_version":"1.0","source":{"id":"2405.01719","kind":"arxiv","version":2}},"canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d7ed06f52f57d4ebfefa3f9cdacf20858f62a7ea01035f6350f91a49d4d1e85e","first_computed_at":"2026-07-05T08:15:50.929994Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:15:50.929994Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fYv3LpznR5g8ZotgcVEVmDFFYXgfO1YpXn18mIb2Vwa6xd4KHK0ZvYMW6783lN7QTyEGjtPXp2a9IoYqXJGDAw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:15:50.930538Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.01719","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c3fe7157c48cad06c852c58c9a4492e72fa24f2eeb4dc028eb573192d752c97e","sha256:e5523c014ab81e427a26a797610bd3ffcc3ac754247d0c56709ebd16cbbc384c"],"state_sha256":"ea863d7fbbbda022c3178b5c2a7f364f74307fa0836ba5722c267e9157811785"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3Gu5SRltx+GvrdHUwjnG6Wou/yMgzkrJN4FzMO8ofTidvYrlfEq0Z4oAbHCCyEOBW9RfVbSTBux43Oeonv2CBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T05:15:33.816437Z","bundle_sha256":"62ee9581079b759b6b69ffc8a78af828d40354083eaa249cba93e32861c6e8c6"}}