{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:JDJWNP4BEXM2S7YNLQYUUIN5MG","short_pith_number":"pith:JDJWNP4B","canonical_record":{"source":{"id":"2308.12539","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-24T03:53:55Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"488eb607e47df8e9293b7d7fcffe37e18ccb43200d54e28a8f6f40ab433c7855","abstract_canon_sha256":"05b56a0b154792acc5ea00d03f49f200f9ce466561061daaa7c8db34a052ab48"},"schema_version":"1.0"},"canonical_sha256":"48d366bf8125d9a97f0d5c314a21bd61a66c5b996b7bbc36559dcce9efc88b33","source":{"kind":"arxiv","id":"2308.12539","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.12539","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"arxiv_version","alias_value":"2308.12539v3","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.12539","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_12","alias_value":"JDJWNP4BEXM2","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_16","alias_value":"JDJWNP4BEXM2S7YN","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_8","alias_value":"JDJWNP4B","created_at":"2026-07-05T08:53:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:JDJWNP4BEXM2S7YNLQYUUIN5MG","target":"record","payload":{"canonical_record":{"source":{"id":"2308.12539","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-24T03:53:55Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"488eb607e47df8e9293b7d7fcffe37e18ccb43200d54e28a8f6f40ab433c7855","abstract_canon_sha256":"05b56a0b154792acc5ea00d03f49f200f9ce466561061daaa7c8db34a052ab48"},"schema_version":"1.0"},"canonical_sha256":"48d366bf8125d9a97f0d5c314a21bd61a66c5b996b7bbc36559dcce9efc88b33","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:53:16.923976Z","signature_b64":"SrkRiPvUifyEitUvPMFej79RtBrIWRlLxLoZEjhLUmpJdzwEyAHNj7km9+maJZqMNQuLY6/MWLFxuoUTclGcBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"48d366bf8125d9a97f0d5c314a21bd61a66c5b996b7bbc36559dcce9efc88b33","last_reissued_at":"2026-07-05T08:53:16.923331Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:53:16.923331Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.12539","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:53:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5aM2WXaAgTMtlEr1emGbL0DPIHQADPezwB9d4+vI5qFV2dQQ08slyFNek4McgE6OO2weLUOsewG0i4XR8qceCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:41:01.469449Z"},"content_sha256":"f15398305a6949bc643d024c8663ca9bc79736e69ae8c932e5f05c5b5c29808d","schema_version":"1.0","event_id":"sha256:f15398305a6949bc643d024c8663ca9bc79736e69ae8c932e5f05c5b5c29808d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:JDJWNP4BEXM2S7YNLQYUUIN5MG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CALM : A Multi-task Benchmark for Comprehensive Assessment of Language Model Bias","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Hugo Lauren\\c{c}on, Pranav Narayanan Venkit, Rebecca J. Passonneau, Shomir Wilson, Vipul Gupta","submitted_at":"2023-08-24T03:53:55Z","abstract_excerpt":"As language models (LMs) become increasingly powerful and widely used, it is important to quantify them for sociodemographic bias with potential for harm. Prior measures of bias are sensitive to perturbations in the templates designed to compare performance across social groups, due to factors such as low diversity or limited number of templates. Also, most previous work considers only one NLP task. We introduce Comprehensive Assessment of Language Models (CALM) for robust measurement of two types of universally relevant sociodemographic bias, gender and race. CALM integrates sixteen datasets "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.12539","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.12539/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:53:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"U6OHP5H3I5dPsUS8z5Vi0gns77B8uvzVoPqFPWm280p5n39IyQ0Mgxh3ilnikMmyxPYuFEDiZOPHcWrOs/xaDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:41:01.470024Z"},"content_sha256":"073739c11c53c16d91eded28d0bb0bdfc85391b27cfda02c6082ca4e89a77021","schema_version":"1.0","event_id":"sha256:073739c11c53c16d91eded28d0bb0bdfc85391b27cfda02c6082ca4e89a77021"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/bundle.json","state_url":"https://pith.science/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T07:41:01Z","links":{"resolver":"https://pith.science/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG","bundle":"https://pith.science/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/bundle.json","state":"https://pith.science/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JDJWNP4BEXM2S7YNLQYUUIN5MG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:JDJWNP4BEXM2S7YNLQYUUIN5MG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"05b56a0b154792acc5ea00d03f49f200f9ce466561061daaa7c8db34a052ab48","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-24T03:53:55Z","title_canon_sha256":"488eb607e47df8e9293b7d7fcffe37e18ccb43200d54e28a8f6f40ab433c7855"},"schema_version":"1.0","source":{"id":"2308.12539","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.12539","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"arxiv_version","alias_value":"2308.12539v3","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.12539","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_12","alias_value":"JDJWNP4BEXM2","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_16","alias_value":"JDJWNP4BEXM2S7YN","created_at":"2026-07-05T08:53:16Z"},{"alias_kind":"pith_short_8","alias_value":"JDJWNP4B","created_at":"2026-07-05T08:53:16Z"}],"graph_snapshots":[{"event_id":"sha256:073739c11c53c16d91eded28d0bb0bdfc85391b27cfda02c6082ca4e89a77021","target":"graph","created_at":"2026-07-05T08:53:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.12539/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As language models (LMs) become increasingly powerful and widely used, it is important to quantify them for sociodemographic bias with potential for harm. Prior measures of bias are sensitive to perturbations in the templates designed to compare performance across social groups, due to factors such as low diversity or limited number of templates. Also, most previous work considers only one NLP task. We introduce Comprehensive Assessment of Language Models (CALM) for robust measurement of two types of universally relevant sociodemographic bias, gender and race. CALM integrates sixteen datasets ","authors_text":"Hugo Lauren\\c{c}on, Pranav Narayanan Venkit, Rebecca J. Passonneau, Shomir Wilson, Vipul Gupta","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-24T03:53:55Z","title":"CALM : A Multi-task Benchmark for Comprehensive Assessment of Language Model Bias"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.12539","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f15398305a6949bc643d024c8663ca9bc79736e69ae8c932e5f05c5b5c29808d","target":"record","created_at":"2026-07-05T08:53:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"05b56a0b154792acc5ea00d03f49f200f9ce466561061daaa7c8db34a052ab48","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-24T03:53:55Z","title_canon_sha256":"488eb607e47df8e9293b7d7fcffe37e18ccb43200d54e28a8f6f40ab433c7855"},"schema_version":"1.0","source":{"id":"2308.12539","kind":"arxiv","version":3}},"canonical_sha256":"48d366bf8125d9a97f0d5c314a21bd61a66c5b996b7bbc36559dcce9efc88b33","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"48d366bf8125d9a97f0d5c314a21bd61a66c5b996b7bbc36559dcce9efc88b33","first_computed_at":"2026-07-05T08:53:16.923331Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:53:16.923331Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"SrkRiPvUifyEitUvPMFej79RtBrIWRlLxLoZEjhLUmpJdzwEyAHNj7km9+maJZqMNQuLY6/MWLFxuoUTclGcBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:53:16.923976Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.12539","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f15398305a6949bc643d024c8663ca9bc79736e69ae8c932e5f05c5b5c29808d","sha256:073739c11c53c16d91eded28d0bb0bdfc85391b27cfda02c6082ca4e89a77021"],"state_sha256":"0ab06812c530a519dcaf80acc513b0c2a5e2717dc6c137b00bceaf22427bcb06"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SHkyM3E/cj03gXlTyFZ7ywgU4rEe76BbgH+IrQovLDMcKAJmxGvuJRcnOdTDbBZ9vMbZ2vV1QpbP+ePn8F0DAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T07:41:01.475381Z","bundle_sha256":"6fa54509e5b55c3f9822c22231f26c49aca01dfe9162afed39590265b4e69365"}}