{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:OFCIYBKBOPJQFV6UEXGHKZZSIG","short_pith_number":"pith:OFCIYBKB","canonical_record":{"source":{"id":"2210.05769","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-10-11T20:19:11Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"4f25f9deeccac6226fc994d39db32a9a15802b6aef8bf48f1e8d72311d197f4b","abstract_canon_sha256":"fa788b9f60a6a133a852e936bcf26b877314615f6831fe9252e0dc82db9f2000"},"schema_version":"1.0"},"canonical_sha256":"71448c054173d302d7d425cc7567324186aeb57dba82bfa048381b07a9a2bf09","source":{"kind":"arxiv","id":"2210.05769","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.05769","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"arxiv_version","alias_value":"2210.05769v3","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.05769","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_12","alias_value":"OFCIYBKBOPJQ","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_16","alias_value":"OFCIYBKBOPJQFV6U","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_8","alias_value":"OFCIYBKB","created_at":"2026-07-05T06:56:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:OFCIYBKBOPJQFV6UEXGHKZZSIG","target":"record","payload":{"canonical_record":{"source":{"id":"2210.05769","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-10-11T20:19:11Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"4f25f9deeccac6226fc994d39db32a9a15802b6aef8bf48f1e8d72311d197f4b","abstract_canon_sha256":"fa788b9f60a6a133a852e936bcf26b877314615f6831fe9252e0dc82db9f2000"},"schema_version":"1.0"},"canonical_sha256":"71448c054173d302d7d425cc7567324186aeb57dba82bfa048381b07a9a2bf09","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:56:30.248894Z","signature_b64":"GXxRFp0bNKWXyPXC23oV3R/gpFnyhQFWB0w+2SiUxfU7UmV4ESoQc0tshSrDQdcP0PpgcEw/xaStHJBCC0FMCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"71448c054173d302d7d425cc7567324186aeb57dba82bfa048381b07a9a2bf09","last_reissued_at":"2026-07-05T06:56:30.248442Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:56:30.248442Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.05769","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:56:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"88lC2fSX3z3GRLrTIdkPrj1AGfivbJs8JW1f6gu2rl3VnuNOtQ3fuVfxdOJ3sO7XSgQMEV0JcbGoNrJYBSM0AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:13:44.588167Z"},"content_sha256":"38b81664248e8fa3413fbb857b083fabc6cc1f95dc2caa4acdb8be81f73b110e","schema_version":"1.0","event_id":"sha256:38b81664248e8fa3413fbb857b083fabc6cc1f95dc2caa4acdb8be81f73b110e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:OFCIYBKBOPJQFV6UEXGHKZZSIG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Vote'n'Rank: Revision of Benchmarking with Social Choice Theory","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Andrey Kravchenko, Daniel Karabekyan, Ekaterina Artemova, Elena Tutubalina, Mark Rofin, Mikhail Florinskiy, Tatiana Shavrina, Vladislav Mikhailov","submitted_at":"2022-10-11T20:19:11Z","abstract_excerpt":"The development of state-of-the-art systems in different applied areas of machine learning (ML) is driven by benchmarks, which have shaped the paradigm of evaluating generalisation capabilities from multiple perspectives. Although the paradigm is shifting towards more fine-grained evaluation across diverse tasks, the delicate question of how to aggregate the performances has received particular interest in the community. In general, benchmarks follow the unspoken utilitarian principles, where the systems are ranked based on their mean average score over task-specific metrics. Such aggregation "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.05769","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.05769/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:56:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Zzldigerua0D2leliyPoRCJRN9keYvO2feleXVzAwRtl0Zj8pkZHtTssctMR8jHVeD9ZyCaIcz/2sM8OQbp5Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:13:44.589108Z"},"content_sha256":"a49fe443400f722345b14f6fb65a8f41a1b39b56c2ac0210729c556c40c6bd5e","schema_version":"1.0","event_id":"sha256:a49fe443400f722345b14f6fb65a8f41a1b39b56c2ac0210729c556c40c6bd5e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/bundle.json","state_url":"https://pith.science/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T01:13:44Z","links":{"resolver":"https://pith.science/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG","bundle":"https://pith.science/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/bundle.json","state":"https://pith.science/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OFCIYBKBOPJQFV6UEXGHKZZSIG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:OFCIYBKBOPJQFV6UEXGHKZZSIG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fa788b9f60a6a133a852e936bcf26b877314615f6831fe9252e0dc82db9f2000","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-10-11T20:19:11Z","title_canon_sha256":"4f25f9deeccac6226fc994d39db32a9a15802b6aef8bf48f1e8d72311d197f4b"},"schema_version":"1.0","source":{"id":"2210.05769","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.05769","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"arxiv_version","alias_value":"2210.05769v3","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.05769","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_12","alias_value":"OFCIYBKBOPJQ","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_16","alias_value":"OFCIYBKBOPJQFV6U","created_at":"2026-07-05T06:56:30Z"},{"alias_kind":"pith_short_8","alias_value":"OFCIYBKB","created_at":"2026-07-05T06:56:30Z"}],"graph_snapshots":[{"event_id":"sha256:a49fe443400f722345b14f6fb65a8f41a1b39b56c2ac0210729c556c40c6bd5e","target":"graph","created_at":"2026-07-05T06:56:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.05769/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The development of state-of-the-art systems in different applied areas of machine learning (ML) is driven by benchmarks, which have shaped the paradigm of evaluating generalisation capabilities from multiple perspectives. Although the paradigm is shifting towards more fine-grained evaluation across diverse tasks, the delicate question of how to aggregate the performances has received particular interest in the community. In general, benchmarks follow the unspoken utilitarian principles, where the systems are ranked based on their mean average score over task-specific metrics. Such aggregation ","authors_text":"Andrey Kravchenko, Daniel Karabekyan, Ekaterina Artemova, Elena Tutubalina, Mark Rofin, Mikhail Florinskiy, Tatiana Shavrina, Vladislav Mikhailov","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-10-11T20:19:11Z","title":"Vote'n'Rank: Revision of Benchmarking with Social Choice Theory"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.05769","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:38b81664248e8fa3413fbb857b083fabc6cc1f95dc2caa4acdb8be81f73b110e","target":"record","created_at":"2026-07-05T06:56:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fa788b9f60a6a133a852e936bcf26b877314615f6831fe9252e0dc82db9f2000","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-10-11T20:19:11Z","title_canon_sha256":"4f25f9deeccac6226fc994d39db32a9a15802b6aef8bf48f1e8d72311d197f4b"},"schema_version":"1.0","source":{"id":"2210.05769","kind":"arxiv","version":3}},"canonical_sha256":"71448c054173d302d7d425cc7567324186aeb57dba82bfa048381b07a9a2bf09","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"71448c054173d302d7d425cc7567324186aeb57dba82bfa048381b07a9a2bf09","first_computed_at":"2026-07-05T06:56:30.248442Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:56:30.248442Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GXxRFp0bNKWXyPXC23oV3R/gpFnyhQFWB0w+2SiUxfU7UmV4ESoQc0tshSrDQdcP0PpgcEw/xaStHJBCC0FMCg==","signature_status":"signed_v1","signed_at":"2026-07-05T06:56:30.248894Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.05769","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:38b81664248e8fa3413fbb857b083fabc6cc1f95dc2caa4acdb8be81f73b110e","sha256:a49fe443400f722345b14f6fb65a8f41a1b39b56c2ac0210729c556c40c6bd5e"],"state_sha256":"18750bfea64f0c42b452c3b9b866f2e2c1816a76e5ddaa5ad92dc361dae9f961"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kLfohQqViKQ9Ing+M84tRir8khKyC0BkpGTcYRf3sX6BQMlrzxlnDkoDR5jxP4XAiMz56xIpb7hZGwClFmxQCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T01:13:44.600359Z","bundle_sha256":"c1badbdf03786c53087226c87b9fd057e20300d1b9863227b38e4046d1d1bac8"}}