{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:64IGK6YMOBK372IVRS6I4VUADE","short_pith_number":"pith:64IGK6YM","canonical_record":{"source":{"id":"2210.16133","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-28T14:01:32Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"ae398daaabdc32349ca8d539cc5c07bba9fa8285fb528ab6d5146146a70964c7","abstract_canon_sha256":"f3fdfdbc69be1a9fa479804a6acd28c77300bb3e68547095d6e6c82336b3a26f"},"schema_version":"1.0"},"canonical_sha256":"f710657b0c7055bfe9158cbc8e5680192468e6a277b8afdecede2b55f27cc60f","source":{"kind":"arxiv","id":"2210.16133","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.16133","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"arxiv_version","alias_value":"2210.16133v2","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.16133","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_12","alias_value":"64IGK6YMOBK3","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_16","alias_value":"64IGK6YMOBK372IV","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_8","alias_value":"64IGK6YM","created_at":"2026-07-05T05:21:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:64IGK6YMOBK372IVRS6I4VUADE","target":"record","payload":{"canonical_record":{"source":{"id":"2210.16133","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-28T14:01:32Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"ae398daaabdc32349ca8d539cc5c07bba9fa8285fb528ab6d5146146a70964c7","abstract_canon_sha256":"f3fdfdbc69be1a9fa479804a6acd28c77300bb3e68547095d6e6c82336b3a26f"},"schema_version":"1.0"},"canonical_sha256":"f710657b0c7055bfe9158cbc8e5680192468e6a277b8afdecede2b55f27cc60f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:21:03.958458Z","signature_b64":"NVZLcHpRl7iPmT4Qw+El7FmNn5tdcsia50YBhA6SF7N36UeWikCCKgHePARBliFuVS/diSP6nMRGDnBm87ZBDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f710657b0c7055bfe9158cbc8e5680192468e6a277b8afdecede2b55f27cc60f","last_reissued_at":"2026-07-05T05:21:03.956483Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:21:03.956483Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.16133","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:21:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hMaRBTxBcbCOPgtZjcD45Gsu0GTJN/vzFCavqQyPJZFiAhr8oTOIP/8HYaAGAU+XU68iTzYDs3AdLAeM9GOwAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T00:27:55.192530Z"},"content_sha256":"1d1357afb1bdadd8a8504a70802fff1827dca401b8ef0084fe75bf4f7e24bcd0","schema_version":"1.0","event_id":"sha256:1d1357afb1bdadd8a8504a70802fff1827dca401b8ef0084fe75bf4f7e24bcd0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:64IGK6YMOBK372IVRS6I4VUADE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Stop Measuring Calibration When Humans Disagree","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Barbara Plank, Joris Baan, Raquel Fern\\'andez, Wilker Aziz","submitted_at":"2022-10-28T14:01:32Z","abstract_excerpt":"Calibration is a popular framework to evaluate whether a classifier knows when it does not know - i.e., its predictive probabilities are a good indication of how likely a prediction is to be correct. Correctness is commonly estimated against the human majority class. Recently, calibration to human majority has been measured on tasks where humans inherently disagree about which class applies. We show that measuring calibration to human majority given inherent disagreements is theoretically problematic, demonstrate this empirically on the ChaosNLI dataset, and derive several instance-level measu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.16133","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.16133/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:21:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iqK32hc1GdcyiDzlr872aJaasXPXCBivolX7iKdsXzzShWg1RK2owlsUls/nChclWewWDuffWfZsGxGoy912Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T00:27:55.193065Z"},"content_sha256":"a06ce704c0ce2c202f45ef1baeb5cf2a6c3702054744f969b75549c56a878072","schema_version":"1.0","event_id":"sha256:a06ce704c0ce2c202f45ef1baeb5cf2a6c3702054744f969b75549c56a878072"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/64IGK6YMOBK372IVRS6I4VUADE/bundle.json","state_url":"https://pith.science/pith/64IGK6YMOBK372IVRS6I4VUADE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/64IGK6YMOBK372IVRS6I4VUADE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T00:27:55Z","links":{"resolver":"https://pith.science/pith/64IGK6YMOBK372IVRS6I4VUADE","bundle":"https://pith.science/pith/64IGK6YMOBK372IVRS6I4VUADE/bundle.json","state":"https://pith.science/pith/64IGK6YMOBK372IVRS6I4VUADE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/64IGK6YMOBK372IVRS6I4VUADE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:64IGK6YMOBK372IVRS6I4VUADE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f3fdfdbc69be1a9fa479804a6acd28c77300bb3e68547095d6e6c82336b3a26f","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-28T14:01:32Z","title_canon_sha256":"ae398daaabdc32349ca8d539cc5c07bba9fa8285fb528ab6d5146146a70964c7"},"schema_version":"1.0","source":{"id":"2210.16133","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.16133","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"arxiv_version","alias_value":"2210.16133v2","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.16133","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_12","alias_value":"64IGK6YMOBK3","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_16","alias_value":"64IGK6YMOBK372IV","created_at":"2026-07-05T05:21:03Z"},{"alias_kind":"pith_short_8","alias_value":"64IGK6YM","created_at":"2026-07-05T05:21:03Z"}],"graph_snapshots":[{"event_id":"sha256:a06ce704c0ce2c202f45ef1baeb5cf2a6c3702054744f969b75549c56a878072","target":"graph","created_at":"2026-07-05T05:21:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.16133/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Calibration is a popular framework to evaluate whether a classifier knows when it does not know - i.e., its predictive probabilities are a good indication of how likely a prediction is to be correct. Correctness is commonly estimated against the human majority class. Recently, calibration to human majority has been measured on tasks where humans inherently disagree about which class applies. We show that measuring calibration to human majority given inherent disagreements is theoretically problematic, demonstrate this empirically on the ChaosNLI dataset, and derive several instance-level measu","authors_text":"Barbara Plank, Joris Baan, Raquel Fern\\'andez, Wilker Aziz","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-28T14:01:32Z","title":"Stop Measuring Calibration When Humans Disagree"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.16133","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1d1357afb1bdadd8a8504a70802fff1827dca401b8ef0084fe75bf4f7e24bcd0","target":"record","created_at":"2026-07-05T05:21:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f3fdfdbc69be1a9fa479804a6acd28c77300bb3e68547095d6e6c82336b3a26f","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-28T14:01:32Z","title_canon_sha256":"ae398daaabdc32349ca8d539cc5c07bba9fa8285fb528ab6d5146146a70964c7"},"schema_version":"1.0","source":{"id":"2210.16133","kind":"arxiv","version":2}},"canonical_sha256":"f710657b0c7055bfe9158cbc8e5680192468e6a277b8afdecede2b55f27cc60f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f710657b0c7055bfe9158cbc8e5680192468e6a277b8afdecede2b55f27cc60f","first_computed_at":"2026-07-05T05:21:03.956483Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:21:03.956483Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NVZLcHpRl7iPmT4Qw+El7FmNn5tdcsia50YBhA6SF7N36UeWikCCKgHePARBliFuVS/diSP6nMRGDnBm87ZBDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T05:21:03.958458Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.16133","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1d1357afb1bdadd8a8504a70802fff1827dca401b8ef0084fe75bf4f7e24bcd0","sha256:a06ce704c0ce2c202f45ef1baeb5cf2a6c3702054744f969b75549c56a878072"],"state_sha256":"4de71343ad99679de41001af63ba182e090ffcc1860be5325d329e7049e32398"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"b/hm286nXu/uuup0XRFIYDSD3pE4so1qKnfRyB8EnrVtt05bWv3Oda112G4CWnjQkKhAb1HAbeF2NNWDIdVLBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T00:27:55.198688Z","bundle_sha256":"26220a92a34d39162ecdac05f4affa302c42bba0e7892f3fd999004bf384587c"}}