{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:U7Q6MDU5NPFBNTOSKQE2S4DN2J","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a1e50670d9a5aad00e09b851b9918d007531c07a2649f997511794d391c19b3b","cross_cats_sorted":["cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T19:57:15Z","title_canon_sha256":"1adb2300c2356ace42fe40478327cc9f878ffbd4194c45bb69c3509f0fe200ca"},"schema_version":"1.0","source":{"id":"2205.11601","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.11601","created_at":"2026-07-05T04:26:05Z"},{"alias_kind":"arxiv_version","alias_value":"2205.11601v1","created_at":"2026-07-05T04:26:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.11601","created_at":"2026-07-05T04:26:05Z"},{"alias_kind":"pith_short_12","alias_value":"U7Q6MDU5NPFB","created_at":"2026-07-05T04:26:05Z"},{"alias_kind":"pith_short_16","alias_value":"U7Q6MDU5NPFBNTOS","created_at":"2026-07-05T04:26:05Z"},{"alias_kind":"pith_short_8","alias_value":"U7Q6MDU5","created_at":"2026-07-05T04:26:05Z"}],"graph_snapshots":[{"event_id":"sha256:ea02526a1dae9f54d1a1e8413c1c8b1d3080b2b7a15a6ffbb43e01b823767ecd","target":"graph","created_at":"2026-07-05T04:26:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2205.11601/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Researchers have devised numerous ways to quantify social biases vested in pretrained language models. As some language models are capable of generating coherent completions given a set of textual prompts, several prompting datasets have been proposed to measure biases between social groups -- posing language generation as a way of identifying biases. In this opinion paper, we analyze how specific choices of prompt sets, metrics, automatic tools and sampling strategies affect bias results. We find out that the practice of measuring biases through text completion is prone to yielding contradict","authors_text":"Afra Feyza Aky\\\"urek, Derry Wijaya, Muhammed Yusuf Kocyigit, Sejin Paik","cross_cats":["cs.CY"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T19:57:15Z","title":"Challenges in Measuring Bias via Open-Ended Language Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.11601","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bd32b5c37a2712b8f58f22bd6da06da02377079f1012510353b081a48a0599c4","target":"record","created_at":"2026-07-05T04:26:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a1e50670d9a5aad00e09b851b9918d007531c07a2649f997511794d391c19b3b","cross_cats_sorted":["cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T19:57:15Z","title_canon_sha256":"1adb2300c2356ace42fe40478327cc9f878ffbd4194c45bb69c3509f0fe200ca"},"schema_version":"1.0","source":{"id":"2205.11601","kind":"arxiv","version":1}},"canonical_sha256":"a7e1e60e9d6bca16cdd25409a9706dd24204483cc7cde6355d1970e57913224c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a7e1e60e9d6bca16cdd25409a9706dd24204483cc7cde6355d1970e57913224c","first_computed_at":"2026-07-05T04:26:05.609924Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:26:05.609924Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9OxNdadh6j/RMAw+LnlLU1g3MgwaKlMGuJp55Gv2TuNiYDoMO7+3brbZVg8+XoiT6xPmbdMwAgW/bT+bigvvDg==","signature_status":"signed_v1","signed_at":"2026-07-05T04:26:05.610396Z","signed_message":"canonical_sha256_bytes"},"source_id":"2205.11601","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bd32b5c37a2712b8f58f22bd6da06da02377079f1012510353b081a48a0599c4","sha256:ea02526a1dae9f54d1a1e8413c1c8b1d3080b2b7a15a6ffbb43e01b823767ecd"],"state_sha256":"c1dc049104c91993cf9703571a085cd7f57e7d8b79b89d2e2c98897168263041"}