{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:MABHCAGCAX3Q5BM5IGFFGWJGU3","short_pith_number":"pith:MABHCAGC","canonical_record":{"source":{"id":"2308.11483","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T14:54:59Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"79e25918fdd6f241e3910fe67d60d3ad502ae442f7db62fb5bc5ed5e11b3df1f","abstract_canon_sha256":"179982832a54ab776cf1a1ae1be84a2f3c6d033e4e76d942942bb6328b390095"},"schema_version":"1.0"},"canonical_sha256":"60027100c205f70e859d418a535926a6e85fa1d3f3fd595544462cef538a8030","source":{"kind":"arxiv","id":"2308.11483","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.11483","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"arxiv_version","alias_value":"2308.11483v1","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.11483","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_12","alias_value":"MABHCAGCAX3Q","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_16","alias_value":"MABHCAGCAX3Q5BM5","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_8","alias_value":"MABHCAGC","created_at":"2026-07-05T06:43:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:MABHCAGCAX3Q5BM5IGFFGWJGU3","target":"record","payload":{"canonical_record":{"source":{"id":"2308.11483","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T14:54:59Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"79e25918fdd6f241e3910fe67d60d3ad502ae442f7db62fb5bc5ed5e11b3df1f","abstract_canon_sha256":"179982832a54ab776cf1a1ae1be84a2f3c6d033e4e76d942942bb6328b390095"},"schema_version":"1.0"},"canonical_sha256":"60027100c205f70e859d418a535926a6e85fa1d3f3fd595544462cef538a8030","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:43:35.539143Z","signature_b64":"T09OVcNqQ8J67Swsih0UGefUhSvz1c9+r+PUH5BusjEWY7uxEQB7q271rgVFR0GkHop+4mA2D1UQsi88aKr0DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"60027100c205f70e859d418a535926a6e85fa1d3f3fd595544462cef538a8030","last_reissued_at":"2026-07-05T06:43:35.538603Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:43:35.538603Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.11483","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:43:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NhgpI3oF96u2uAQTfx9l9bS4RM+MKKcqJ33ymhoPJzK0sw/dH7BMD0z6M3QQNFWTzjMzQzcObtF+1xezlgNzCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T08:14:54.612341Z"},"content_sha256":"7940f25d7841868d9eb452104d3db1ab37719cb5cc7c3ef289a2679bd02aa60d","schema_version":"1.0","event_id":"sha256:7940f25d7841868d9eb452104d3db1ab37719cb5cc7c3ef289a2679bd02aa60d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:MABHCAGCAX3Q5BM5IGFFGWJGU3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Large Language Models Sensitivity to The Order of Options in Multiple-Choice Questions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Estevam Hruschka, Pouya Pezeshkpour","submitted_at":"2023-08-22T14:54:59Z","abstract_excerpt":"Large Language Models (LLMs) have demonstrated remarkable capabilities in various NLP tasks. However, previous works have shown these models are sensitive towards prompt wording, and few-shot demonstrations and their order, posing challenges to fair assessment of these models. As these models become more powerful, it becomes imperative to understand and address these limitations. In this paper, we focus on LLMs robustness on the task of multiple-choice questions -- commonly adopted task to study reasoning and fact-retrieving capability of LLMs. Investigating the sensitivity of LLMs towards the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.11483","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.11483/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:43:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9MhR72yOCGyY/ocAgMKATfjXlOApZRB632GumNNuDc+uMpDA/EVGNmiwnUKwGHRHRga3omTZ21kJsOe+MH9wAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T08:14:54.612852Z"},"content_sha256":"7077392541b043742f24530e4cb5f25672c3ff2d9189f0aa5d5e3003692552a0","schema_version":"1.0","event_id":"sha256:7077392541b043742f24530e4cb5f25672c3ff2d9189f0aa5d5e3003692552a0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/bundle.json","state_url":"https://pith.science/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T08:14:54Z","links":{"resolver":"https://pith.science/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3","bundle":"https://pith.science/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/bundle.json","state":"https://pith.science/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MABHCAGCAX3Q5BM5IGFFGWJGU3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:MABHCAGCAX3Q5BM5IGFFGWJGU3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"179982832a54ab776cf1a1ae1be84a2f3c6d033e4e76d942942bb6328b390095","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T14:54:59Z","title_canon_sha256":"79e25918fdd6f241e3910fe67d60d3ad502ae442f7db62fb5bc5ed5e11b3df1f"},"schema_version":"1.0","source":{"id":"2308.11483","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.11483","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"arxiv_version","alias_value":"2308.11483v1","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.11483","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_12","alias_value":"MABHCAGCAX3Q","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_16","alias_value":"MABHCAGCAX3Q5BM5","created_at":"2026-07-05T06:43:35Z"},{"alias_kind":"pith_short_8","alias_value":"MABHCAGC","created_at":"2026-07-05T06:43:35Z"}],"graph_snapshots":[{"event_id":"sha256:7077392541b043742f24530e4cb5f25672c3ff2d9189f0aa5d5e3003692552a0","target":"graph","created_at":"2026-07-05T06:43:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.11483/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have demonstrated remarkable capabilities in various NLP tasks. However, previous works have shown these models are sensitive towards prompt wording, and few-shot demonstrations and their order, posing challenges to fair assessment of these models. As these models become more powerful, it becomes imperative to understand and address these limitations. In this paper, we focus on LLMs robustness on the task of multiple-choice questions -- commonly adopted task to study reasoning and fact-retrieving capability of LLMs. Investigating the sensitivity of LLMs towards the","authors_text":"Estevam Hruschka, Pouya Pezeshkpour","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T14:54:59Z","title":"Large Language Models Sensitivity to The Order of Options in Multiple-Choice Questions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.11483","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7940f25d7841868d9eb452104d3db1ab37719cb5cc7c3ef289a2679bd02aa60d","target":"record","created_at":"2026-07-05T06:43:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"179982832a54ab776cf1a1ae1be84a2f3c6d033e4e76d942942bb6328b390095","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T14:54:59Z","title_canon_sha256":"79e25918fdd6f241e3910fe67d60d3ad502ae442f7db62fb5bc5ed5e11b3df1f"},"schema_version":"1.0","source":{"id":"2308.11483","kind":"arxiv","version":1}},"canonical_sha256":"60027100c205f70e859d418a535926a6e85fa1d3f3fd595544462cef538a8030","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"60027100c205f70e859d418a535926a6e85fa1d3f3fd595544462cef538a8030","first_computed_at":"2026-07-05T06:43:35.538603Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:43:35.538603Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"T09OVcNqQ8J67Swsih0UGefUhSvz1c9+r+PUH5BusjEWY7uxEQB7q271rgVFR0GkHop+4mA2D1UQsi88aKr0DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:43:35.539143Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.11483","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7940f25d7841868d9eb452104d3db1ab37719cb5cc7c3ef289a2679bd02aa60d","sha256:7077392541b043742f24530e4cb5f25672c3ff2d9189f0aa5d5e3003692552a0"],"state_sha256":"27e9f41f15d4cb548a450ef89c5da76d6051d01f4f3a8e3138d6a5105d0e09f6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Y0W+Omnyx1B85SvPmzecW7OYdQQe2zww9XsxqaLFjfJY6hctdH34biqvp4vnhx+irZ9/fhOE/5io/w3NPK1mBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T08:14:54.617737Z","bundle_sha256":"fa6af36a40f315acf10e1f8ee0053de976101fa4dc773b649bb8a517e645d117"}}