{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:GYTFJTRHHWIQQJREHN5BQWKBQS","short_pith_number":"pith:GYTFJTRH","canonical_record":{"source":{"id":"2305.12757","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T06:28:48Z","cross_cats_sorted":[],"title_canon_sha256":"e6441082cfaa192befff19f5e164009a9a508eec74adc431dca4c6a1e2a4d7bf","abstract_canon_sha256":"a9c2578c47195fad3a67503cd48e1a94efb7f77a34502d4dfcbd4b62ca961288"},"schema_version":"1.0"},"canonical_sha256":"362654ce273d910826243b7a18594184a076916b284a6b6830e1219de637727d","source":{"kind":"arxiv","id":"2305.12757","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.12757","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"arxiv_version","alias_value":"2305.12757v1","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.12757","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_12","alias_value":"GYTFJTRHHWIQ","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_16","alias_value":"GYTFJTRHHWIQQJRE","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_8","alias_value":"GYTFJTRH","created_at":"2026-07-05T06:12:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:GYTFJTRHHWIQQJREHN5BQWKBQS","target":"record","payload":{"canonical_record":{"source":{"id":"2305.12757","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T06:28:48Z","cross_cats_sorted":[],"title_canon_sha256":"e6441082cfaa192befff19f5e164009a9a508eec74adc431dca4c6a1e2a4d7bf","abstract_canon_sha256":"a9c2578c47195fad3a67503cd48e1a94efb7f77a34502d4dfcbd4b62ca961288"},"schema_version":"1.0"},"canonical_sha256":"362654ce273d910826243b7a18594184a076916b284a6b6830e1219de637727d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:12:14.053783Z","signature_b64":"XW1BU3EakogB3s/4yBoRF/vMr7DmjXOh9jy7fDuaz5zP3g9gbyXg25rr11k45ZF8qJrRpjjHw9Mz7Tt/ynA0Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"362654ce273d910826243b7a18594184a076916b284a6b6830e1219de637727d","last_reissued_at":"2026-07-05T06:12:14.053390Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:12:14.053390Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.12757","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RZ/Wf8RWXHfd7x94/qtUuoCvzNlfk2xA9dm1i0SuVnh1tMbynnA2zZXJSsK7p5Wfs+ebt5YRu6jxWgtc3NqXDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T20:19:32.859172Z"},"content_sha256":"71d1848789408e002d913fd2aa0482e68dd3a484e630f4589bcf55d88dc71836","schema_version":"1.0","event_id":"sha256:71d1848789408e002d913fd2aa0482e68dd3a484e630f4589bcf55d88dc71836"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:GYTFJTRHHWIQQJREHN5BQWKBQS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"This Prompt is Measuring <MASK>: Evaluating Bias Evaluation in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Eddie Ungless, Esma Balkir, Seraphina Goldfarb-Tarrant, Su Lin Blodgett","submitted_at":"2023-05-22T06:28:48Z","abstract_excerpt":"Bias research in NLP seeks to analyse models for social biases, thus helping NLP practitioners uncover, measure, and mitigate social harms. We analyse the body of work that uses prompts and templates to assess bias in language models. We draw on a measurement modelling framework to create a taxonomy of attributes that capture what a bias test aims to measure and how that measurement is carried out. By applying this taxonomy to 90 bias tests, we illustrate qualitatively and quantitatively that core aspects of bias test conceptualisations and operationalisations are frequently unstated or ambigu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.12757","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.12757/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:12:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4vplqolcBOh5ptTqeqvtGEk4KSMTcEs4DNg89TPhI+n1d7Mt91yJ7T9+4Fzbav7lXkEmos4TbY/r/igDG/lpCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T20:19:32.860073Z"},"content_sha256":"2b17fd0c0f68b65f0914f8ffdd933a0d515513735a04d9f43309d06f311cdc57","schema_version":"1.0","event_id":"sha256:2b17fd0c0f68b65f0914f8ffdd933a0d515513735a04d9f43309d06f311cdc57"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/bundle.json","state_url":"https://pith.science/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T20:19:32Z","links":{"resolver":"https://pith.science/pith/GYTFJTRHHWIQQJREHN5BQWKBQS","bundle":"https://pith.science/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/bundle.json","state":"https://pith.science/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GYTFJTRHHWIQQJREHN5BQWKBQS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:GYTFJTRHHWIQQJREHN5BQWKBQS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a9c2578c47195fad3a67503cd48e1a94efb7f77a34502d4dfcbd4b62ca961288","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T06:28:48Z","title_canon_sha256":"e6441082cfaa192befff19f5e164009a9a508eec74adc431dca4c6a1e2a4d7bf"},"schema_version":"1.0","source":{"id":"2305.12757","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.12757","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"arxiv_version","alias_value":"2305.12757v1","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.12757","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_12","alias_value":"GYTFJTRHHWIQ","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_16","alias_value":"GYTFJTRHHWIQQJRE","created_at":"2026-07-05T06:12:14Z"},{"alias_kind":"pith_short_8","alias_value":"GYTFJTRH","created_at":"2026-07-05T06:12:14Z"}],"graph_snapshots":[{"event_id":"sha256:2b17fd0c0f68b65f0914f8ffdd933a0d515513735a04d9f43309d06f311cdc57","target":"graph","created_at":"2026-07-05T06:12:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.12757/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Bias research in NLP seeks to analyse models for social biases, thus helping NLP practitioners uncover, measure, and mitigate social harms. We analyse the body of work that uses prompts and templates to assess bias in language models. We draw on a measurement modelling framework to create a taxonomy of attributes that capture what a bias test aims to measure and how that measurement is carried out. By applying this taxonomy to 90 bias tests, we illustrate qualitatively and quantitatively that core aspects of bias test conceptualisations and operationalisations are frequently unstated or ambigu","authors_text":"Eddie Ungless, Esma Balkir, Seraphina Goldfarb-Tarrant, Su Lin Blodgett","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T06:28:48Z","title":"This Prompt is Measuring <MASK>: Evaluating Bias Evaluation in Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.12757","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:71d1848789408e002d913fd2aa0482e68dd3a484e630f4589bcf55d88dc71836","target":"record","created_at":"2026-07-05T06:12:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a9c2578c47195fad3a67503cd48e1a94efb7f77a34502d4dfcbd4b62ca961288","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T06:28:48Z","title_canon_sha256":"e6441082cfaa192befff19f5e164009a9a508eec74adc431dca4c6a1e2a4d7bf"},"schema_version":"1.0","source":{"id":"2305.12757","kind":"arxiv","version":1}},"canonical_sha256":"362654ce273d910826243b7a18594184a076916b284a6b6830e1219de637727d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"362654ce273d910826243b7a18594184a076916b284a6b6830e1219de637727d","first_computed_at":"2026-07-05T06:12:14.053390Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:12:14.053390Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XW1BU3EakogB3s/4yBoRF/vMr7DmjXOh9jy7fDuaz5zP3g9gbyXg25rr11k45ZF8qJrRpjjHw9Mz7Tt/ynA0Dw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:12:14.053783Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.12757","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:71d1848789408e002d913fd2aa0482e68dd3a484e630f4589bcf55d88dc71836","sha256:2b17fd0c0f68b65f0914f8ffdd933a0d515513735a04d9f43309d06f311cdc57"],"state_sha256":"72a402e7d93cfa7e1bba5eb2b4a3e8695684f4878a1890cde11c933cf43a7270"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iWqIb1uWUVNSMANkrERkk1dk4aUw5T6OnpGN7/HKfkdeWXwUf6UnR0ZyqMUkcg/zDhPjTBiz+Fq9U1UoDYk7Cg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T20:19:32.865906Z","bundle_sha256":"8d9197cb1ec4e24d3e56e15e071f3d10fa92a9a733a91035240cbdfd836fffd7"}}