{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:VE6J43XA7IQSYEQM62JKAIJQB3","short_pith_number":"pith:VE6J43XA","canonical_record":{"source":{"id":"2305.11921","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ME","submitted_at":"2023-05-19T08:58:55Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"1931d48336e31832c515875c05d9b5b9f59968c5b7dbb4a878b72fc6b61a5ca4","abstract_canon_sha256":"8493e2bc5c1a70ef8913985e56397dec0f17adb4fad978764691319b940bd533"},"schema_version":"1.0"},"canonical_sha256":"a93c9e6ee0fa212c120cf692a021300ee38a4b042e0acc869f4aa08bef520b56","source":{"kind":"arxiv","id":"2305.11921","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.11921","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"arxiv_version","alias_value":"2305.11921v1","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.11921","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_12","alias_value":"VE6J43XA7IQS","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_16","alias_value":"VE6J43XA7IQSYEQM","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_8","alias_value":"VE6J43XA","created_at":"2026-07-05T06:12:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:VE6J43XA7IQSYEQM62JKAIJQB3","target":"record","payload":{"canonical_record":{"source":{"id":"2305.11921","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ME","submitted_at":"2023-05-19T08:58:55Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"1931d48336e31832c515875c05d9b5b9f59968c5b7dbb4a878b72fc6b61a5ca4","abstract_canon_sha256":"8493e2bc5c1a70ef8913985e56397dec0f17adb4fad978764691319b940bd533"},"schema_version":"1.0"},"canonical_sha256":"a93c9e6ee0fa212c120cf692a021300ee38a4b042e0acc869f4aa08bef520b56","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:12:04.544209Z","signature_b64":"T3eliVss1dHolPHX5NDIoptr3NGqVt4qtLV0W7L5DXBbd0icVv2UGaS4t7sGk4Pg0NROitGpNpzBmLrqFiO4Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a93c9e6ee0fa212c120cf692a021300ee38a4b042e0acc869f4aa08bef520b56","last_reissued_at":"2026-07-05T06:12:04.543813Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:12:04.543813Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.11921","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5nNN0jglpE/UTGiBqIF+YRqVvltYZRQYAiRwISJX0cfZ97r2lIshlUnNkf+QiANzqVsRWJBd9F8m0RE/iq7RCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T10:14:23.009861Z"},"content_sha256":"5770228daa95092c05c4ccfbd490399d15917cda35df65da72361a0cdbcb7ffb","schema_version":"1.0","event_id":"sha256:5770228daa95092c05c4ccfbd490399d15917cda35df65da72361a0cdbcb7ffb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:VE6J43XA7IQSYEQM62JKAIJQB3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An Approach to Multiple Comparison Benchmark Evaluations that is Stable Under Manipulation of the Comparate Set","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.PF"],"primary_cat":"stat.ME","authors_text":"Ali Ismail-Fawaz, Angus Dempster, Chang Wei Tan, Daniel F. Schmidt, Geoffrey I. Webb, Germain Forestier, Jonathan Weber, Lynn Miller, Matthieu Herrmann, Maxime Devanne, Stefano Berretti","submitted_at":"2023-05-19T08:58:55Z","abstract_excerpt":"The measurement of progress using benchmarks evaluations is ubiquitous in computer science and machine learning. However, common approaches to analyzing and presenting the results of benchmark comparisons of multiple algorithms over multiple datasets, such as the critical difference diagram introduced by Dem\\v{s}ar (2006), have important shortcomings and, we show, are open to both inadvertent and intentional manipulation. To address these issues, we propose a new approach to presenting the results of benchmark comparisons, the Multiple Comparison Matrix (MCM), that prioritizes pairwise compari"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.11921","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.11921/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z1ofm6IxGP0HS4Mz5C38FnA5cZBFPKqf95V+ATpfN5JijNr+KlhhKBCH9rqFXOSAx5R7+UFyzjC7hxu81VEZCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T10:14:23.010199Z"},"content_sha256":"599b2a03aa5f2a6766d145826bcde0fc49a7203258205a9f7ade61a126fa6e69","schema_version":"1.0","event_id":"sha256:599b2a03aa5f2a6766d145826bcde0fc49a7203258205a9f7ade61a126fa6e69"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VE6J43XA7IQSYEQM62JKAIJQB3/bundle.json","state_url":"https://pith.science/pith/VE6J43XA7IQSYEQM62JKAIJQB3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VE6J43XA7IQSYEQM62JKAIJQB3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T10:14:23Z","links":{"resolver":"https://pith.science/pith/VE6J43XA7IQSYEQM62JKAIJQB3","bundle":"https://pith.science/pith/VE6J43XA7IQSYEQM62JKAIJQB3/bundle.json","state":"https://pith.science/pith/VE6J43XA7IQSYEQM62JKAIJQB3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VE6J43XA7IQSYEQM62JKAIJQB3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:VE6J43XA7IQSYEQM62JKAIJQB3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8493e2bc5c1a70ef8913985e56397dec0f17adb4fad978764691319b940bd533","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ME","submitted_at":"2023-05-19T08:58:55Z","title_canon_sha256":"1931d48336e31832c515875c05d9b5b9f59968c5b7dbb4a878b72fc6b61a5ca4"},"schema_version":"1.0","source":{"id":"2305.11921","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.11921","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"arxiv_version","alias_value":"2305.11921v1","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.11921","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_12","alias_value":"VE6J43XA7IQS","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_16","alias_value":"VE6J43XA7IQSYEQM","created_at":"2026-07-05T06:12:04Z"},{"alias_kind":"pith_short_8","alias_value":"VE6J43XA","created_at":"2026-07-05T06:12:04Z"}],"graph_snapshots":[{"event_id":"sha256:599b2a03aa5f2a6766d145826bcde0fc49a7203258205a9f7ade61a126fa6e69","target":"graph","created_at":"2026-07-05T06:12:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.11921/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The measurement of progress using benchmarks evaluations is ubiquitous in computer science and machine learning. However, common approaches to analyzing and presenting the results of benchmark comparisons of multiple algorithms over multiple datasets, such as the critical difference diagram introduced by Dem\\v{s}ar (2006), have important shortcomings and, we show, are open to both inadvertent and intentional manipulation. To address these issues, we propose a new approach to presenting the results of benchmark comparisons, the Multiple Comparison Matrix (MCM), that prioritizes pairwise compari","authors_text":"Ali Ismail-Fawaz, Angus Dempster, Chang Wei Tan, Daniel F. Schmidt, Geoffrey I. Webb, Germain Forestier, Jonathan Weber, Lynn Miller, Matthieu Herrmann, Maxime Devanne, Stefano Berretti","cross_cats":["cs.AI","cs.LG","cs.PF"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ME","submitted_at":"2023-05-19T08:58:55Z","title":"An Approach to Multiple Comparison Benchmark Evaluations that is Stable Under Manipulation of the Comparate Set"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.11921","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5770228daa95092c05c4ccfbd490399d15917cda35df65da72361a0cdbcb7ffb","target":"record","created_at":"2026-07-05T06:12:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8493e2bc5c1a70ef8913985e56397dec0f17adb4fad978764691319b940bd533","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ME","submitted_at":"2023-05-19T08:58:55Z","title_canon_sha256":"1931d48336e31832c515875c05d9b5b9f59968c5b7dbb4a878b72fc6b61a5ca4"},"schema_version":"1.0","source":{"id":"2305.11921","kind":"arxiv","version":1}},"canonical_sha256":"a93c9e6ee0fa212c120cf692a021300ee38a4b042e0acc869f4aa08bef520b56","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a93c9e6ee0fa212c120cf692a021300ee38a4b042e0acc869f4aa08bef520b56","first_computed_at":"2026-07-05T06:12:04.543813Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:12:04.543813Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"T3eliVss1dHolPHX5NDIoptr3NGqVt4qtLV0W7L5DXBbd0icVv2UGaS4t7sGk4Pg0NROitGpNpzBmLrqFiO4Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T06:12:04.544209Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.11921","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5770228daa95092c05c4ccfbd490399d15917cda35df65da72361a0cdbcb7ffb","sha256:599b2a03aa5f2a6766d145826bcde0fc49a7203258205a9f7ade61a126fa6e69"],"state_sha256":"26c4e458dedc34d7f88285635ed122ce70c795aa964d4df18b409639da517f13"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iHWZeY9CxUVTJHbVhkoblPncWiqpnnXiaccQx2Hs5Y7Ao9XzevqevsddYOjaEv5v1GNzczZ1+53ccXEgw5kSAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T10:14:23.012706Z","bundle_sha256":"dc2150951436618764cb6b15ec8377da3d0053e7e3920fb965ce5af579285517"}}