{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RGP3HIGJEXG6HA4FSJX3KTSC2R","short_pith_number":"pith:RGP3HIGJ","schema_version":"1.0","canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","source":{"kind":"arxiv","id":"2407.15018","version":2},"attestation_state":"computed","paper":{"title":"Answer, Assemble, Ace: Understanding How LMs Answer Multiple Choice Questions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ashish Sabharwal, Hannaneh Hajishirzi, Oyvind Tafjord, Sarah Wiegreffe, Yonatan Belinkov","submitted_at":"2024-07-21T00:10:23Z","abstract_excerpt":"Multiple-choice question answering (MCQA) is a key competence of performant transformer language models that is tested by mainstream benchmarks. However, recent evidence shows that models can have quite a range of performance, particularly when the task format is diversified slightly (such as by shuffling answer choice order). In this work we ask: how do successful models perform formatted MCQA? We employ vocabulary projection and activation patching methods to localize key hidden states that encode relevant information for predicting the correct answer. We find that the prediction of a specif"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2407.15018","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"37f549b6b470a0f6a258f71d8a27d944a7d5f344ca40a33e5c75700dbca8f255","abstract_canon_sha256":"358bf5cb8db2b36a8661750ac94e1db1682c6aa814b24009b781bc12e3fb302d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:26:38.796520Z","signature_b64":"RavrA18I82lsO5yxfQb5lwrslmflB/H3ou14r16lDKyYGewIyZo+tyyQYgdSxCMIwVjztDneNXINjPc8SyXBBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","last_reissued_at":"2026-07-05T10:26:38.796008Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:26:38.796008Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Answer, Assemble, Ace: Understanding How LMs Answer Multiple Choice Questions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ashish Sabharwal, Hannaneh Hajishirzi, Oyvind Tafjord, Sarah Wiegreffe, Yonatan Belinkov","submitted_at":"2024-07-21T00:10:23Z","abstract_excerpt":"Multiple-choice question answering (MCQA) is a key competence of performant transformer language models that is tested by mainstream benchmarks. However, recent evidence shows that models can have quite a range of performance, particularly when the task format is diversified slightly (such as by shuffling answer choice order). In this work we ask: how do successful models perform formatted MCQA? We employ vocabulary projection and activation patching methods to localize key hidden states that encode relevant information for predicting the correct answer. We find that the prediction of a specif"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.15018","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.15018/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2407.15018","created_at":"2026-07-05T10:26:38.796073+00:00"},{"alias_kind":"arxiv_version","alias_value":"2407.15018v2","created_at":"2026-07-05T10:26:38.796073+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.15018","created_at":"2026-07-05T10:26:38.796073+00:00"},{"alias_kind":"pith_short_12","alias_value":"RGP3HIGJEXG6","created_at":"2026-07-05T10:26:38.796073+00:00"},{"alias_kind":"pith_short_16","alias_value":"RGP3HIGJEXG6HA4F","created_at":"2026-07-05T10:26:38.796073+00:00"},{"alias_kind":"pith_short_8","alias_value":"RGP3HIGJ","created_at":"2026-07-05T10:26:38.796073+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.11643","citing_title":"Improving Cross-Format Robustness in Language Models with Multi-Format Training","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2606.00640","citing_title":"An Attribute-Based Measure of Video Complexity","ref_index":55,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12813","citing_title":"REALISTA: Realistic Latent Adversarial Attacks that Elicit LLM Hallucinations","ref_index":153,"is_internal_anchor":false},{"citing_arxiv_id":"2604.06854","citing_title":"To Adapt or not to Adapt, Rethinking the Value of Medical Knowledge-Aware Large Language Models","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R","json":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R.json","graph_json":"https://pith.science/api/pith-number/RGP3HIGJEXG6HA4FSJX3KTSC2R/graph.json","events_json":"https://pith.science/api/pith-number/RGP3HIGJEXG6HA4FSJX3KTSC2R/events.json","paper":"https://pith.science/paper/RGP3HIGJ"},"agent_actions":{"view_html":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R","download_json":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R.json","view_paper":"https://pith.science/paper/RGP3HIGJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2407.15018&json=true","fetch_graph":"https://pith.science/api/pith-number/RGP3HIGJEXG6HA4FSJX3KTSC2R/graph.json","fetch_events":"https://pith.science/api/pith-number/RGP3HIGJEXG6HA4FSJX3KTSC2R/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/action/storage_attestation","attest_author":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/action/author_attestation","sign_citation":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/action/citation_signature","submit_replication":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/action/replication_record"}},"created_at":"2026-07-05T10:26:38.796073+00:00","updated_at":"2026-07-05T10:26:38.796073+00:00"}