{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:2MS3UF56I7GLLV4Q6KBZANQMEM","short_pith_number":"pith:2MS3UF56","schema_version":"1.0","canonical_sha256":"d325ba17be47ccb5d790f28390360c231b6a03b569db6a2e3460dd1b8268fc7b","source":{"kind":"arxiv","id":"2309.03882","version":4},"attestation_state":"computed","paper":{"title":"Large Language Models Are Not Robust Multiple Choice Selectors","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chujie Zheng, Fandong Meng, Hao Zhou, Jie Zhou, Minlie Huang","submitted_at":"2023-09-07T17:44:56Z","abstract_excerpt":"Multiple choice questions (MCQs) serve as a common yet important task format in the evaluation of large language models (LLMs). This work shows that modern LLMs are vulnerable to option position changes in MCQs due to their inherent \"selection bias\", namely, they prefer to select specific option IDs as answers (like \"Option A\"). Through extensive empirical analyses with 20 LLMs on three benchmarks, we pinpoint that this behavioral bias primarily stems from LLMs' token bias, where the model a priori assigns more probabilistic mass to specific option ID tokens (e.g., A/B/C/D) when predicting ans"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.03882","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-07T17:44:56Z","cross_cats_sorted":[],"title_canon_sha256":"6d23c3187ffeb38f39c693931985734b22ff346c27a05da0db57fbd491628aa4","abstract_canon_sha256":"88cccd65e89a364427a0c0d3423aade40b129b9346b821375311237abfff8ece"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:47:59.110325Z","signature_b64":"jrBai1IAY3n/T3P+zxL5KUWkKQqPb1MjSGKxu5EAaHu/U9+s/MRJxNrbhHtmbqJNkzaRrtwthBl7gfa/93OOAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d325ba17be47ccb5d790f28390360c231b6a03b569db6a2e3460dd1b8268fc7b","last_reissued_at":"2026-07-05T07:47:59.109798Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:47:59.109798Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Large Language Models Are Not Robust Multiple Choice Selectors","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chujie Zheng, Fandong Meng, Hao Zhou, Jie Zhou, Minlie Huang","submitted_at":"2023-09-07T17:44:56Z","abstract_excerpt":"Multiple choice questions (MCQs) serve as a common yet important task format in the evaluation of large language models (LLMs). This work shows that modern LLMs are vulnerable to option position changes in MCQs due to their inherent \"selection bias\", namely, they prefer to select specific option IDs as answers (like \"Option A\"). Through extensive empirical analyses with 20 LLMs on three benchmarks, we pinpoint that this behavioral bias primarily stems from LLMs' token bias, where the model a priori assigns more probabilistic mass to specific option ID tokens (e.g., A/B/C/D) when predicting ans"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.03882","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.03882/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.03882","created_at":"2026-07-05T07:47:59.109854+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.03882v4","created_at":"2026-07-05T07:47:59.109854+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.03882","created_at":"2026-07-05T07:47:59.109854+00:00"},{"alias_kind":"pith_short_12","alias_value":"2MS3UF56I7GL","created_at":"2026-07-05T07:47:59.109854+00:00"},{"alias_kind":"pith_short_16","alias_value":"2MS3UF56I7GLLV4Q","created_at":"2026-07-05T07:47:59.109854+00:00"},{"alias_kind":"pith_short_8","alias_value":"2MS3UF56","created_at":"2026-07-05T07:47:59.109854+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":23,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.08065","citing_title":"When LLMs Agree, Are They Right? Auditing Self-Consistency and Cross-Model Agreement as Confidence Signals","ref_index":60,"is_internal_anchor":true},{"citing_arxiv_id":"2606.22470","citing_title":"PRIME: Evaluating Prompt Resolution Under Incompatible Instructions in LLMs","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08034","citing_title":"Sci-Rho: A Multilingual Visually-Grounded Symbolic Benchmark for STEM Problems","ref_index":77,"is_internal_anchor":false},{"citing_arxiv_id":"2607.00664","citing_title":"YOMI-Bench: A Benchmark for Evaluating Kanji Reading and Phonological Understanding of LLMs for Japanese","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04273","citing_title":"Human agency in initial human-AI proof formalization workflows","ref_index":230,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24703","citing_title":"TS-Skill: A Benchmark for Evaluating Analytical Skills in Time-Series Question Answering","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25492","citing_title":"SafetyRepro: Configuration-Conditional Rank Instability on Alignment Benchmarks","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2601.12805","citing_title":"SciHorizon-GENE: Benchmarking LLM for Life Sciences Inference from Gene Knowledge to Functional Understanding","ref_index":70,"is_internal_anchor":false},{"citing_arxiv_id":"2509.19590","citing_title":"Position: AI Evaluations Should be Grounded on a Theory of Capability","ref_index":55,"is_internal_anchor":false},{"citing_arxiv_id":"2511.14159","citing_title":"MVI-Bench: A Comprehensive Benchmark for Evaluating Robustness to Misleading Visual Inputs in LVLMs","ref_index":67,"is_internal_anchor":false},{"citing_arxiv_id":"2605.16301","citing_title":"MANTA: Multi-turn Assessment for Nonhuman Thinking & Alignment","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15589","citing_title":"MHGraphBench: Knowledge Graph-Grounded Benchmarking of Mental Health Knowledge in Large Language Models","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19099","citing_title":"DecisionBench: A Benchmark for Emergent Delegation in Long-Horizon Agentic Workflows","ref_index":46,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13045","citing_title":"Large Language Models Lack Temporal Awareness of Medical Knowledge","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11591","citing_title":"Logit-Attention Divergence: Mitigating Position Bias in Multi-Image Retrieval via Attention-Guided Calibration","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27249","citing_title":"Instruction Complexity Induces Positional Collapse in Adversarial LLM Evaluation","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08614","citing_title":"DiagnosticIQ: A Benchmark for LLM-Based Industrial Maintenance Action Recommendation from Symbolic Rules","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10808","citing_title":"Threat Modelling using Domain-Adapted Language Models: Empirical Evaluation and Insights","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06161","citing_title":"Beyond Accuracy: Policy Invariance as a Reliability Test for LLM Safety Judges","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10034","citing_title":"AI Achieves a Perfect LSAT Score","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16902","citing_title":"Beyond Text-Dominance: Understanding Modality Preference of Omni-modal Large Language Models","ref_index":47,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01846","citing_title":"Do Large Language Models Plan Answer Positions? Position Bias in Multiple-Choice Question Generation","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00907","citing_title":"TRIP-Evaluate: An Open Multimodal Benchmark for Evaluating Large Models in Transportation","ref_index":31,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM","json":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM.json","graph_json":"https://pith.science/api/pith-number/2MS3UF56I7GLLV4Q6KBZANQMEM/graph.json","events_json":"https://pith.science/api/pith-number/2MS3UF56I7GLLV4Q6KBZANQMEM/events.json","paper":"https://pith.science/paper/2MS3UF56"},"agent_actions":{"view_html":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM","download_json":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM.json","view_paper":"https://pith.science/paper/2MS3UF56","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.03882&json=true","fetch_graph":"https://pith.science/api/pith-number/2MS3UF56I7GLLV4Q6KBZANQMEM/graph.json","fetch_events":"https://pith.science/api/pith-number/2MS3UF56I7GLLV4Q6KBZANQMEM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM/action/storage_attestation","attest_author":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM/action/author_attestation","sign_citation":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM/action/citation_signature","submit_replication":"https://pith.science/pith/2MS3UF56I7GLLV4Q6KBZANQMEM/action/replication_record"}},"created_at":"2026-07-05T07:47:59.109854+00:00","updated_at":"2026-07-05T07:47:59.109854+00:00"}