{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:RGP3HIGJEXG6HA4FSJX3KTSC2R","short_pith_number":"pith:RGP3HIGJ","canonical_record":{"source":{"id":"2407.15018","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"37f549b6b470a0f6a258f71d8a27d944a7d5f344ca40a33e5c75700dbca8f255","abstract_canon_sha256":"358bf5cb8db2b36a8661750ac94e1db1682c6aa814b24009b781bc12e3fb302d"},"schema_version":"1.0"},"canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","source":{"kind":"arxiv","id":"2407.15018","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.15018","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"arxiv_version","alias_value":"2407.15018v2","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.15018","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_12","alias_value":"RGP3HIGJEXG6","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_16","alias_value":"RGP3HIGJEXG6HA4F","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_8","alias_value":"RGP3HIGJ","created_at":"2026-07-05T10:26:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:RGP3HIGJEXG6HA4FSJX3KTSC2R","target":"record","payload":{"canonical_record":{"source":{"id":"2407.15018","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"37f549b6b470a0f6a258f71d8a27d944a7d5f344ca40a33e5c75700dbca8f255","abstract_canon_sha256":"358bf5cb8db2b36a8661750ac94e1db1682c6aa814b24009b781bc12e3fb302d"},"schema_version":"1.0"},"canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:26:38.796520Z","signature_b64":"RavrA18I82lsO5yxfQb5lwrslmflB/H3ou14r16lDKyYGewIyZo+tyyQYgdSxCMIwVjztDneNXINjPc8SyXBBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","last_reissued_at":"2026-07-05T10:26:38.796008Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:26:38.796008Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.15018","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:26:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mVvxrx5UcMLWxq0SeGpW+NRHJ6xNKG99NpxcD3cglrzTsSciDCcjZfAP+9jcCnsRt5sJcBGeQw3lhU7EPcSRCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:59:02.489926Z"},"content_sha256":"1057d513367bc23796f1b3dc50b6b88ed1b6473b9fc96bceaabdc3fae2bde948","schema_version":"1.0","event_id":"sha256:1057d513367bc23796f1b3dc50b6b88ed1b6473b9fc96bceaabdc3fae2bde948"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:RGP3HIGJEXG6HA4FSJX3KTSC2R","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Answer, Assemble, Ace: Understanding How LMs Answer Multiple Choice Questions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ashish Sabharwal, Hannaneh Hajishirzi, Oyvind Tafjord, Sarah Wiegreffe, Yonatan Belinkov","submitted_at":"2024-07-21T00:10:23Z","abstract_excerpt":"Multiple-choice question answering (MCQA) is a key competence of performant transformer language models that is tested by mainstream benchmarks. However, recent evidence shows that models can have quite a range of performance, particularly when the task format is diversified slightly (such as by shuffling answer choice order). In this work we ask: how do successful models perform formatted MCQA? We employ vocabulary projection and activation patching methods to localize key hidden states that encode relevant information for predicting the correct answer. We find that the prediction of a specif"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.15018","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.15018/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:26:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bWjdePUry2T/8KFp0zFzSCQghZJb4wr4yti2/6eLJAZydCKEw90fJggqyWo0gY564szOYMhka8szdLIK8jKfBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:59:02.490555Z"},"content_sha256":"33ad75326c3e65d050af6ac1b950fb54459cd5c381983d93e24f3c90e34718ed","schema_version":"1.0","event_id":"sha256:33ad75326c3e65d050af6ac1b950fb54459cd5c381983d93e24f3c90e34718ed"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/bundle.json","state_url":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T17:59:02Z","links":{"resolver":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R","bundle":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/bundle.json","state":"https://pith.science/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RGP3HIGJEXG6HA4FSJX3KTSC2R/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:RGP3HIGJEXG6HA4FSJX3KTSC2R","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"358bf5cb8db2b36a8661750ac94e1db1682c6aa814b24009b781bc12e3fb302d","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","title_canon_sha256":"37f549b6b470a0f6a258f71d8a27d944a7d5f344ca40a33e5c75700dbca8f255"},"schema_version":"1.0","source":{"id":"2407.15018","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.15018","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"arxiv_version","alias_value":"2407.15018v2","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.15018","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_12","alias_value":"RGP3HIGJEXG6","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_16","alias_value":"RGP3HIGJEXG6HA4F","created_at":"2026-07-05T10:26:38Z"},{"alias_kind":"pith_short_8","alias_value":"RGP3HIGJ","created_at":"2026-07-05T10:26:38Z"}],"graph_snapshots":[{"event_id":"sha256:33ad75326c3e65d050af6ac1b950fb54459cd5c381983d93e24f3c90e34718ed","target":"graph","created_at":"2026-07-05T10:26:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.15018/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multiple-choice question answering (MCQA) is a key competence of performant transformer language models that is tested by mainstream benchmarks. However, recent evidence shows that models can have quite a range of performance, particularly when the task format is diversified slightly (such as by shuffling answer choice order). In this work we ask: how do successful models perform formatted MCQA? We employ vocabulary projection and activation patching methods to localize key hidden states that encode relevant information for predicting the correct answer. We find that the prediction of a specif","authors_text":"Ashish Sabharwal, Hannaneh Hajishirzi, Oyvind Tafjord, Sarah Wiegreffe, Yonatan Belinkov","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","title":"Answer, Assemble, Ace: Understanding How LMs Answer Multiple Choice Questions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.15018","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1057d513367bc23796f1b3dc50b6b88ed1b6473b9fc96bceaabdc3fae2bde948","target":"record","created_at":"2026-07-05T10:26:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"358bf5cb8db2b36a8661750ac94e1db1682c6aa814b24009b781bc12e3fb302d","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-21T00:10:23Z","title_canon_sha256":"37f549b6b470a0f6a258f71d8a27d944a7d5f344ca40a33e5c75700dbca8f255"},"schema_version":"1.0","source":{"id":"2407.15018","kind":"arxiv","version":2}},"canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"899fb3a0c925cde38385926fb54e42d46a450713d64e733ca3d1d84be5cac87d","first_computed_at":"2026-07-05T10:26:38.796008Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:26:38.796008Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"RavrA18I82lsO5yxfQb5lwrslmflB/H3ou14r16lDKyYGewIyZo+tyyQYgdSxCMIwVjztDneNXINjPc8SyXBBw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:26:38.796520Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.15018","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1057d513367bc23796f1b3dc50b6b88ed1b6473b9fc96bceaabdc3fae2bde948","sha256:33ad75326c3e65d050af6ac1b950fb54459cd5c381983d93e24f3c90e34718ed"],"state_sha256":"5529d81b686184d5cf6de190f95c0d4a3c11ef17beb8bc890f6fb4eefdb14306"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Q3d8UJgPEvE2yN6X/AM/Vx/Z/DElT5ZpMVWmf6QsDyhYhPn150k1hI5L688/QPQilzUCFi5/d9e0NvC286dpAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T17:59:02.497076Z","bundle_sha256":"91ec2e5ecb98345b07687e22aea8cc8707f7b0012efc5203c786720b53de6e2a"}}