{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:6ACCDOT3GAO45FLBMTYE6VMWHM","short_pith_number":"pith:6ACCDOT3","canonical_record":{"source":{"id":"2608.02966","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-03T23:59:39Z","cross_cats_sorted":[],"title_canon_sha256":"536648abbe1a3c7a30acfaf0acb306d521fc4a100e19ad295d09c27145441be7","abstract_canon_sha256":"f99572b96811e8db6c04f9289bee077eb70137dfbf45da4a886a4b8e5f1608ff"},"schema_version":"1.0"},"canonical_sha256":"f00421ba7b301dce956164f04f55963b37b460c72eaa15d5046129a7ad7bac95","source":{"kind":"arxiv","id":"2608.02966","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2608.02966","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"arxiv_version","alias_value":"2608.02966v1","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2608.02966","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_12","alias_value":"6ACCDOT3GAO4","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_16","alias_value":"6ACCDOT3GAO45FLB","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_8","alias_value":"6ACCDOT3","created_at":"2026-08-05T00:44:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:6ACCDOT3GAO45FLBMTYE6VMWHM","target":"record","payload":{"canonical_record":{"source":{"id":"2608.02966","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-03T23:59:39Z","cross_cats_sorted":[],"title_canon_sha256":"536648abbe1a3c7a30acfaf0acb306d521fc4a100e19ad295d09c27145441be7","abstract_canon_sha256":"f99572b96811e8db6c04f9289bee077eb70137dfbf45da4a886a4b8e5f1608ff"},"schema_version":"1.0"},"canonical_sha256":"f00421ba7b301dce956164f04f55963b37b460c72eaa15d5046129a7ad7bac95","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-08-05T00:44:16.996200Z","signature_b64":"CN1ya859vYsJN4XSux0Xe+9/8lFOV+7REq8i/FxMpHuvH3M9OOaT+pyyWoydtrSsDD0EFY3vQkJBR1af8vy/DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f00421ba7b301dce956164f04f55963b37b460c72eaa15d5046129a7ad7bac95","last_reissued_at":"2026-08-05T00:44:16.993868Z","signature_status":"signed_v1","first_computed_at":"2026-08-05T00:44:16.993868Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2608.02966","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-08-05T00:44:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Go1y8odytXxI/YnMC+f5hmsFGeGjvnh3ybIDZ1F6qwjp1uvzrxnJ17f2xif3U9vyEvW5HZUImtCbCkrKwQTABA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T01:53:58.732371Z"},"content_sha256":"1527332f4303282b744e643add5a71fc15ced7e689f0742e230a992f84e4c923","schema_version":"1.0","event_id":"sha256:1527332f4303282b744e643add5a71fc15ced7e689f0742e230a992f84e4c923"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:6ACCDOT3GAO45FLBMTYE6VMWHM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Michalis Vazirgiannis, Sarah Almeida Carneiro, Xiao Fei, Yang Zhang","submitted_at":"2026-08-03T23:59:39Z","abstract_excerpt":"Most multiple-choice question (MCQ) benchmarks evaluate Large Language Models (LLMs) only by whether they select the correct answers. This binary scoring treats all incorrect responses alike, even though an LLM's preferences among incorrect options may contain systematic and useful information about its behavior and ability. We introduce the LLM Nominal Response Model (LLM-NRM), an option-aware psychometric framework that models the full distribution over answer choices to jointly estimate LLM ability and option-level item characteristics, while separating model-specific response calibration s"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2608.02966","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2608.02966/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-08-05T00:44:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Jdcqg6m2zvx26fIQsfHU0bOmQPqH2d2hwLeqs9vn4Fg3mhlVO0dsF6VkM05lilOf7E1f4urMMs9LTgmLz3WeAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T01:53:58.732869Z"},"content_sha256":"c61ce5ed505b9ad57be4b563c159ed29244538966135a1d27ecfdd9f7274de76","schema_version":"1.0","event_id":"sha256:c61ce5ed505b9ad57be4b563c159ed29244538966135a1d27ecfdd9f7274de76"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/bundle.json","state_url":"https://pith.science/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T01:53:58Z","links":{"resolver":"https://pith.science/pith/6ACCDOT3GAO45FLBMTYE6VMWHM","bundle":"https://pith.science/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/bundle.json","state":"https://pith.science/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6ACCDOT3GAO45FLBMTYE6VMWHM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:6ACCDOT3GAO45FLBMTYE6VMWHM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f99572b96811e8db6c04f9289bee077eb70137dfbf45da4a886a4b8e5f1608ff","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-03T23:59:39Z","title_canon_sha256":"536648abbe1a3c7a30acfaf0acb306d521fc4a100e19ad295d09c27145441be7"},"schema_version":"1.0","source":{"id":"2608.02966","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2608.02966","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"arxiv_version","alias_value":"2608.02966v1","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2608.02966","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_12","alias_value":"6ACCDOT3GAO4","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_16","alias_value":"6ACCDOT3GAO45FLB","created_at":"2026-08-05T00:44:16Z"},{"alias_kind":"pith_short_8","alias_value":"6ACCDOT3","created_at":"2026-08-05T00:44:16Z"}],"graph_snapshots":[{"event_id":"sha256:c61ce5ed505b9ad57be4b563c159ed29244538966135a1d27ecfdd9f7274de76","target":"graph","created_at":"2026-08-05T00:44:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2608.02966/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most multiple-choice question (MCQ) benchmarks evaluate Large Language Models (LLMs) only by whether they select the correct answers. This binary scoring treats all incorrect responses alike, even though an LLM's preferences among incorrect options may contain systematic and useful information about its behavior and ability. We introduce the LLM Nominal Response Model (LLM-NRM), an option-aware psychometric framework that models the full distribution over answer choices to jointly estimate LLM ability and option-level item characteristics, while separating model-specific response calibration s","authors_text":"Michalis Vazirgiannis, Sarah Almeida Carneiro, Xiao Fei, Yang Zhang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-03T23:59:39Z","title":"Every Wrong Answer Counts: Option-Level Psychometrics for LLM Multiple-Choice Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2608.02966","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1527332f4303282b744e643add5a71fc15ced7e689f0742e230a992f84e4c923","target":"record","created_at":"2026-08-05T00:44:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f99572b96811e8db6c04f9289bee077eb70137dfbf45da4a886a4b8e5f1608ff","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-03T23:59:39Z","title_canon_sha256":"536648abbe1a3c7a30acfaf0acb306d521fc4a100e19ad295d09c27145441be7"},"schema_version":"1.0","source":{"id":"2608.02966","kind":"arxiv","version":1}},"canonical_sha256":"f00421ba7b301dce956164f04f55963b37b460c72eaa15d5046129a7ad7bac95","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f00421ba7b301dce956164f04f55963b37b460c72eaa15d5046129a7ad7bac95","first_computed_at":"2026-08-05T00:44:16.993868Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-08-05T00:44:16.993868Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"CN1ya859vYsJN4XSux0Xe+9/8lFOV+7REq8i/FxMpHuvH3M9OOaT+pyyWoydtrSsDD0EFY3vQkJBR1af8vy/DA==","signature_status":"signed_v1","signed_at":"2026-08-05T00:44:16.996200Z","signed_message":"canonical_sha256_bytes"},"source_id":"2608.02966","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1527332f4303282b744e643add5a71fc15ced7e689f0742e230a992f84e4c923","sha256:c61ce5ed505b9ad57be4b563c159ed29244538966135a1d27ecfdd9f7274de76"],"state_sha256":"7bed1898920e8edad73e6e1432c897a31ca4a1f47f9a6782077f7da22a80d564"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/5k0TFDFwMB1Xt/tZVmkoiUJVuu51lNPOodjyk40DKojyNJiL9a9SpeL6sj77pKr4PWGIiFvwZXnYVvefFuRBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T01:53:58.737845Z","bundle_sha256":"2cc1affe89b63bc72770d71b54916861ea222985f967c9bdf35547e25c7926e2"}}