{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:2MS3UF56I7GLLV4Q6KBZANQMEM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"88cccd65e89a364427a0c0d3423aade40b129b9346b821375311237abfff8ece","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-07T17:44:56Z","title_canon_sha256":"6d23c3187ffeb38f39c693931985734b22ff346c27a05da0db57fbd491628aa4"},"schema_version":"1.0","source":{"id":"2309.03882","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.03882","created_at":"2026-07-05T07:47:59Z"},{"alias_kind":"arxiv_version","alias_value":"2309.03882v4","created_at":"2026-07-05T07:47:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.03882","created_at":"2026-07-05T07:47:59Z"},{"alias_kind":"pith_short_12","alias_value":"2MS3UF56I7GL","created_at":"2026-07-05T07:47:59Z"},{"alias_kind":"pith_short_16","alias_value":"2MS3UF56I7GLLV4Q","created_at":"2026-07-05T07:47:59Z"},{"alias_kind":"pith_short_8","alias_value":"2MS3UF56","created_at":"2026-07-05T07:47:59Z"}],"graph_snapshots":[{"event_id":"sha256:c34b6f2f7402e88a0c0a2636434e7555447cf28904191096bd05be82dbb13da3","target":"graph","created_at":"2026-07-05T07:47:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.03882/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multiple choice questions (MCQs) serve as a common yet important task format in the evaluation of large language models (LLMs). This work shows that modern LLMs are vulnerable to option position changes in MCQs due to their inherent \"selection bias\", namely, they prefer to select specific option IDs as answers (like \"Option A\"). Through extensive empirical analyses with 20 LLMs on three benchmarks, we pinpoint that this behavioral bias primarily stems from LLMs' token bias, where the model a priori assigns more probabilistic mass to specific option ID tokens (e.g., A/B/C/D) when predicting ans","authors_text":"Chujie Zheng, Fandong Meng, Hao Zhou, Jie Zhou, Minlie Huang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-07T17:44:56Z","title":"Large Language Models Are Not Robust Multiple Choice Selectors"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.03882","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7aa9eb77dcf7d7703979c3cd1d1bece9198512b5d37076d3b8fe3a2556c8f26c","target":"record","created_at":"2026-07-05T07:47:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"88cccd65e89a364427a0c0d3423aade40b129b9346b821375311237abfff8ece","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-07T17:44:56Z","title_canon_sha256":"6d23c3187ffeb38f39c693931985734b22ff346c27a05da0db57fbd491628aa4"},"schema_version":"1.0","source":{"id":"2309.03882","kind":"arxiv","version":4}},"canonical_sha256":"d325ba17be47ccb5d790f28390360c231b6a03b569db6a2e3460dd1b8268fc7b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d325ba17be47ccb5d790f28390360c231b6a03b569db6a2e3460dd1b8268fc7b","first_computed_at":"2026-07-05T07:47:59.109798Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:47:59.109798Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jrBai1IAY3n/T3P+zxL5KUWkKQqPb1MjSGKxu5EAaHu/U9+s/MRJxNrbhHtmbqJNkzaRrtwthBl7gfa/93OOAg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:47:59.110325Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.03882","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7aa9eb77dcf7d7703979c3cd1d1bece9198512b5d37076d3b8fe3a2556c8f26c","sha256:c34b6f2f7402e88a0c0a2636434e7555447cf28904191096bd05be82dbb13da3"],"state_sha256":"8a3293c65f34e55eeac48000045eae8245ac9e3090a6432d077c9b6c2c9d9d2e"}