{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:JFOXLAX65JAYW6NUOX24KHTX4L","short_pith_number":"pith:JFOXLAX6","canonical_record":{"source":{"id":"2108.03588","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-08-08T08:32:15Z","cross_cats_sorted":["stat.ME"],"title_canon_sha256":"e897f9d8904b77dcc8a9a2af53bb221728c9a778ca4576188a3e65fbacef51e3","abstract_canon_sha256":"38eb2454bf827a969cb2725450a1dc49a9cf497963608d289a339a15223de56f"},"schema_version":"1.0"},"canonical_sha256":"495d7582feea418b79b475f5c51e77e2fa662c14d0953a341fc1b14e3385654b","source":{"kind":"arxiv","id":"2108.03588","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2108.03588","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"arxiv_version","alias_value":"2108.03588v1","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.03588","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_12","alias_value":"JFOXLAX65JAY","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_16","alias_value":"JFOXLAX65JAYW6NU","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_8","alias_value":"JFOXLAX6","created_at":"2026-07-05T03:04:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:JFOXLAX65JAYW6NUOX24KHTX4L","target":"record","payload":{"canonical_record":{"source":{"id":"2108.03588","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-08-08T08:32:15Z","cross_cats_sorted":["stat.ME"],"title_canon_sha256":"e897f9d8904b77dcc8a9a2af53bb221728c9a778ca4576188a3e65fbacef51e3","abstract_canon_sha256":"38eb2454bf827a969cb2725450a1dc49a9cf497963608d289a339a15223de56f"},"schema_version":"1.0"},"canonical_sha256":"495d7582feea418b79b475f5c51e77e2fa662c14d0953a341fc1b14e3385654b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:04:07.295903Z","signature_b64":"X5rrcI4n58BLp+6P84i1FgP3XfvJgntbrGLxXi3VzMO91I4nQDTg2e/q2cREzXmS2cpKpqSnhDuK5i1zSUecAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"495d7582feea418b79b475f5c51e77e2fa662c14d0953a341fc1b14e3385654b","last_reissued_at":"2026-07-05T03:04:07.295453Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:04:07.295453Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2108.03588","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:04:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gA4im6tl+IthhNxs1TZbBk6W7xXfv5zJiyzs1BoRambBaUVWu9GVRvyhtdRT61aHNWVYsVQQ4WB4s3B7ZF3iBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T18:27:33.772031Z"},"content_sha256":"b8a61c7b390510da6fa190c2d440a22bcf98667b43f935110920eb10b306e20b","schema_version":"1.0","event_id":"sha256:b8a61c7b390510da6fa190c2d440a22bcf98667b43f935110920eb10b306e20b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:JFOXLAX65JAYW6NUOX24KHTX4L","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Look at the Evaluation Setup of the M5 Forecasting Competition","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ME"],"primary_cat":"cs.LG","authors_text":"Christoph Bergmeir, Hansika Hewamalage, Pablo Montero-Manso, Rob J Hyndman","submitted_at":"2021-08-08T08:32:15Z","abstract_excerpt":"Forecast evaluation plays a key role in how empirical evidence shapes the development of the discipline. Domain experts are interested in error measures relevant for their decision making needs. Such measures may produce unreliable results. Although reliability properties of several metrics have already been discussed, it has hardly been quantified in an objective way. We propose a measure named Rank Stability, which evaluates how much the rankings of an experiment differ in between similar datasets, when the models and errors are constant. We use this to study the evaluation setup of the M5. "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.03588","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2108.03588/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:04:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FOL77f+LLMK7/t1hL0ui5tkPlTeZbWEFi1J6MKci31wQ7YZdZPR0aAeUScezeo3o0hBCMOujC3mW2lggVtcMCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T18:27:33.772569Z"},"content_sha256":"72d50693817cc2449aa4b3d93633c65896dbf28a0f8d8db24b3004492c1ec102","schema_version":"1.0","event_id":"sha256:72d50693817cc2449aa4b3d93633c65896dbf28a0f8d8db24b3004492c1ec102"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JFOXLAX65JAYW6NUOX24KHTX4L/bundle.json","state_url":"https://pith.science/pith/JFOXLAX65JAYW6NUOX24KHTX4L/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JFOXLAX65JAYW6NUOX24KHTX4L/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T18:27:33Z","links":{"resolver":"https://pith.science/pith/JFOXLAX65JAYW6NUOX24KHTX4L","bundle":"https://pith.science/pith/JFOXLAX65JAYW6NUOX24KHTX4L/bundle.json","state":"https://pith.science/pith/JFOXLAX65JAYW6NUOX24KHTX4L/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JFOXLAX65JAYW6NUOX24KHTX4L/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:JFOXLAX65JAYW6NUOX24KHTX4L","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"38eb2454bf827a969cb2725450a1dc49a9cf497963608d289a339a15223de56f","cross_cats_sorted":["stat.ME"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-08-08T08:32:15Z","title_canon_sha256":"e897f9d8904b77dcc8a9a2af53bb221728c9a778ca4576188a3e65fbacef51e3"},"schema_version":"1.0","source":{"id":"2108.03588","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2108.03588","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"arxiv_version","alias_value":"2108.03588v1","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.03588","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_12","alias_value":"JFOXLAX65JAY","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_16","alias_value":"JFOXLAX65JAYW6NU","created_at":"2026-07-05T03:04:07Z"},{"alias_kind":"pith_short_8","alias_value":"JFOXLAX6","created_at":"2026-07-05T03:04:07Z"}],"graph_snapshots":[{"event_id":"sha256:72d50693817cc2449aa4b3d93633c65896dbf28a0f8d8db24b3004492c1ec102","target":"graph","created_at":"2026-07-05T03:04:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2108.03588/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Forecast evaluation plays a key role in how empirical evidence shapes the development of the discipline. Domain experts are interested in error measures relevant for their decision making needs. Such measures may produce unreliable results. Although reliability properties of several metrics have already been discussed, it has hardly been quantified in an objective way. We propose a measure named Rank Stability, which evaluates how much the rankings of an experiment differ in between similar datasets, when the models and errors are constant. We use this to study the evaluation setup of the M5. ","authors_text":"Christoph Bergmeir, Hansika Hewamalage, Pablo Montero-Manso, Rob J Hyndman","cross_cats":["stat.ME"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-08-08T08:32:15Z","title":"A Look at the Evaluation Setup of the M5 Forecasting Competition"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.03588","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b8a61c7b390510da6fa190c2d440a22bcf98667b43f935110920eb10b306e20b","target":"record","created_at":"2026-07-05T03:04:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"38eb2454bf827a969cb2725450a1dc49a9cf497963608d289a339a15223de56f","cross_cats_sorted":["stat.ME"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-08-08T08:32:15Z","title_canon_sha256":"e897f9d8904b77dcc8a9a2af53bb221728c9a778ca4576188a3e65fbacef51e3"},"schema_version":"1.0","source":{"id":"2108.03588","kind":"arxiv","version":1}},"canonical_sha256":"495d7582feea418b79b475f5c51e77e2fa662c14d0953a341fc1b14e3385654b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"495d7582feea418b79b475f5c51e77e2fa662c14d0953a341fc1b14e3385654b","first_computed_at":"2026-07-05T03:04:07.295453Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:04:07.295453Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X5rrcI4n58BLp+6P84i1FgP3XfvJgntbrGLxXi3VzMO91I4nQDTg2e/q2cREzXmS2cpKpqSnhDuK5i1zSUecAw==","signature_status":"signed_v1","signed_at":"2026-07-05T03:04:07.295903Z","signed_message":"canonical_sha256_bytes"},"source_id":"2108.03588","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b8a61c7b390510da6fa190c2d440a22bcf98667b43f935110920eb10b306e20b","sha256:72d50693817cc2449aa4b3d93633c65896dbf28a0f8d8db24b3004492c1ec102"],"state_sha256":"bd17d221838ec98906f0cef9299a7b69179987b2d3831e681654ffdac7c5a512"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9zpEJRpW0RghODw7WQuTYm8OWZ/9vyDYVUnHH+JTbT49LzV+k07ciGt63U2/DX/UYopp7iltofDr/P64xF8CCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T18:27:33.778338Z","bundle_sha256":"b43e6d12e05e6322cca17211c96aeb366d7959746a2c6075ff0b5593442a9011"}}