{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:INMB5WAMSFTB2KGTYIFENDC223","short_pith_number":"pith:INMB5WAM","canonical_record":{"source":{"id":"2404.08382","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-12T10:36:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5164ab7f590806223cbc098c3e0c0b1b6e5d60c9b43d5799095a89c8cb64cda4","abstract_canon_sha256":"2f5ee27bf08da52557ff989ddad06929e69a2d3dbbfcf6e37e8fea0ff2c74a9b"},"schema_version":"1.0"},"canonical_sha256":"43581ed80c91661d28d3c20a468c5ad6c6ce305891bcf85cff7a243e6706a83c","source":{"kind":"arxiv","id":"2404.08382","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.08382","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"arxiv_version","alias_value":"2404.08382v2","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.08382","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_12","alias_value":"INMB5WAMSFTB","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_16","alias_value":"INMB5WAMSFTB2KGT","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_8","alias_value":"INMB5WAM","created_at":"2026-07-05T08:57:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:INMB5WAMSFTB2KGTYIFENDC223","target":"record","payload":{"canonical_record":{"source":{"id":"2404.08382","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-12T10:36:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5164ab7f590806223cbc098c3e0c0b1b6e5d60c9b43d5799095a89c8cb64cda4","abstract_canon_sha256":"2f5ee27bf08da52557ff989ddad06929e69a2d3dbbfcf6e37e8fea0ff2c74a9b"},"schema_version":"1.0"},"canonical_sha256":"43581ed80c91661d28d3c20a468c5ad6c6ce305891bcf85cff7a243e6706a83c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:57:02.249304Z","signature_b64":"bzytkEFxUL4DiIFi53y2dE72oJz+ddPQ3WW0XZg2SH1UcMGmilpdtz+qJY90XmuGonYJjXzXlRPHj/GlSmpxCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"43581ed80c91661d28d3c20a468c5ad6c6ce305891bcf85cff7a243e6706a83c","last_reissued_at":"2026-07-05T08:57:02.248896Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:57:02.248896Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.08382","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eFs0wtJso+v7QjYZe62Io91PgzRMuv1pcvSgpDIiK/XjoUIiYgy0BtlGcXDbjIrnIGcF9MliubZJRUfHjA1uBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T23:46:34.405861Z"},"content_sha256":"32e25b5cb27b49dcc8768a74fca29f5eadf1640f8a45f8016ffcc2530a2c9120","schema_version":"1.0","event_id":"sha256:32e25b5cb27b49dcc8768a74fca29f5eadf1640f8a45f8016ffcc2530a2c9120"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:INMB5WAMSFTB2KGTYIFENDC223","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Barbara Plank, Bolei Ma, Chengzhi Hu, Paul R\\\"ottger, Xinpeng Wang","submitted_at":"2024-04-12T10:36:15Z","abstract_excerpt":"Multiple choice questions (MCQs) are commonly used to evaluate the capabilities of large language models (LLMs). One common way to evaluate the model response is to rank the candidate answers based on the log probability of the first token prediction. An alternative way is to examine the text output. Prior work has shown that first token probabilities lack robustness to changes in MCQ phrasing, and that first token probabilities do not match text answers for instruction-tuned models. Therefore, in this paper, we investigate the robustness of text answers. We show that the text answers are more"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.08382","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.08382/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QsCMbiNO9E4GBnXxCebYU06LYNasvAvDZeJF05vZZG0zDxsz0E7j7tGTdKyNV09L+ZpjQJX8fwyjjKNPe/BqBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T23:46:34.406382Z"},"content_sha256":"4e509aa9ca60ca86a05e8cd93cc11f65766b0a233563d0ee37f6d19823837460","schema_version":"1.0","event_id":"sha256:4e509aa9ca60ca86a05e8cd93cc11f65766b0a233563d0ee37f6d19823837460"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/INMB5WAMSFTB2KGTYIFENDC223/bundle.json","state_url":"https://pith.science/pith/INMB5WAMSFTB2KGTYIFENDC223/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/INMB5WAMSFTB2KGTYIFENDC223/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T23:46:34Z","links":{"resolver":"https://pith.science/pith/INMB5WAMSFTB2KGTYIFENDC223","bundle":"https://pith.science/pith/INMB5WAMSFTB2KGTYIFENDC223/bundle.json","state":"https://pith.science/pith/INMB5WAMSFTB2KGTYIFENDC223/state.json","well_known_bundle":"https://pith.science/.well-known/pith/INMB5WAMSFTB2KGTYIFENDC223/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:INMB5WAMSFTB2KGTYIFENDC223","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2f5ee27bf08da52557ff989ddad06929e69a2d3dbbfcf6e37e8fea0ff2c74a9b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-12T10:36:15Z","title_canon_sha256":"5164ab7f590806223cbc098c3e0c0b1b6e5d60c9b43d5799095a89c8cb64cda4"},"schema_version":"1.0","source":{"id":"2404.08382","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.08382","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"arxiv_version","alias_value":"2404.08382v2","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.08382","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_12","alias_value":"INMB5WAMSFTB","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_16","alias_value":"INMB5WAMSFTB2KGT","created_at":"2026-07-05T08:57:02Z"},{"alias_kind":"pith_short_8","alias_value":"INMB5WAM","created_at":"2026-07-05T08:57:02Z"}],"graph_snapshots":[{"event_id":"sha256:4e509aa9ca60ca86a05e8cd93cc11f65766b0a233563d0ee37f6d19823837460","target":"graph","created_at":"2026-07-05T08:57:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.08382/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multiple choice questions (MCQs) are commonly used to evaluate the capabilities of large language models (LLMs). One common way to evaluate the model response is to rank the candidate answers based on the log probability of the first token prediction. An alternative way is to examine the text output. Prior work has shown that first token probabilities lack robustness to changes in MCQ phrasing, and that first token probabilities do not match text answers for instruction-tuned models. Therefore, in this paper, we investigate the robustness of text answers. We show that the text answers are more","authors_text":"Barbara Plank, Bolei Ma, Chengzhi Hu, Paul R\\\"ottger, Xinpeng Wang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-12T10:36:15Z","title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.08382","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:32e25b5cb27b49dcc8768a74fca29f5eadf1640f8a45f8016ffcc2530a2c9120","target":"record","created_at":"2026-07-05T08:57:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2f5ee27bf08da52557ff989ddad06929e69a2d3dbbfcf6e37e8fea0ff2c74a9b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-12T10:36:15Z","title_canon_sha256":"5164ab7f590806223cbc098c3e0c0b1b6e5d60c9b43d5799095a89c8cb64cda4"},"schema_version":"1.0","source":{"id":"2404.08382","kind":"arxiv","version":2}},"canonical_sha256":"43581ed80c91661d28d3c20a468c5ad6c6ce305891bcf85cff7a243e6706a83c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"43581ed80c91661d28d3c20a468c5ad6c6ce305891bcf85cff7a243e6706a83c","first_computed_at":"2026-07-05T08:57:02.248896Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:57:02.248896Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"bzytkEFxUL4DiIFi53y2dE72oJz+ddPQ3WW0XZg2SH1UcMGmilpdtz+qJY90XmuGonYJjXzXlRPHj/GlSmpxCw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:57:02.249304Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.08382","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:32e25b5cb27b49dcc8768a74fca29f5eadf1640f8a45f8016ffcc2530a2c9120","sha256:4e509aa9ca60ca86a05e8cd93cc11f65766b0a233563d0ee37f6d19823837460"],"state_sha256":"b27244ff8c40c52a6753116930854a9402b283fac4a921e4f219612cd007e364"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mams1QQczZXUuAiQ1CMtPVy2btiZibpi1TIlpIKSVZR8oLHx4FprQU6FWhtc5yoounpCMUbvOeIgaWW3CYrzAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T23:46:34.410063Z","bundle_sha256":"2a08b22c720ad7b5ba1b6fd99e6c0fb75ef815fc4185432dbadd7b891fc9e35f"}}