{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:OSFDBSGKGQQTCTQQ4I2CK26OAG","short_pith_number":"pith:OSFDBSGK","canonical_record":{"source":{"id":"2402.14499","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T12:47:33Z","cross_cats_sorted":[],"title_canon_sha256":"b73bd8b535ef2e6b8a14fd6475a142229a719bbf27494853a4c2c50886004634","abstract_canon_sha256":"17aaa71d5aab516edb5c61de8c67aa66ce95d25f6db8a02bcde3c62cc329fe8b"},"schema_version":"1.0"},"canonical_sha256":"748a30c8ca3421314e10e234256bce01ae294fdc42c46a0cc434ba38c610c563","source":{"kind":"arxiv","id":"2402.14499","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.14499","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"arxiv_version","alias_value":"2402.14499v2","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.14499","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_12","alias_value":"OSFDBSGKGQQT","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_16","alias_value":"OSFDBSGKGQQTCTQQ","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_8","alias_value":"OSFDBSGK","created_at":"2026-07-05T08:40:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:OSFDBSGKGQQTCTQQ4I2CK26OAG","target":"record","payload":{"canonical_record":{"source":{"id":"2402.14499","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T12:47:33Z","cross_cats_sorted":[],"title_canon_sha256":"b73bd8b535ef2e6b8a14fd6475a142229a719bbf27494853a4c2c50886004634","abstract_canon_sha256":"17aaa71d5aab516edb5c61de8c67aa66ce95d25f6db8a02bcde3c62cc329fe8b"},"schema_version":"1.0"},"canonical_sha256":"748a30c8ca3421314e10e234256bce01ae294fdc42c46a0cc434ba38c610c563","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:40:13.905723Z","signature_b64":"NKJX38ODURDOwlkWUQsZ+QeuzUf1AaldbFciaHrgEbhdbZh66JrYcJkSXCvcUYFOxiGn6vLhEVrswhjAL0GuCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"748a30c8ca3421314e10e234256bce01ae294fdc42c46a0cc434ba38c610c563","last_reissued_at":"2026-07-05T08:40:13.905075Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:40:13.905075Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.14499","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FD9eAHx7qzkltkpN9Uc3rG24x7bOxNveD0+iUbILKy4n+K/27MA0Z+gWd+VQKipQkeqFntB0YsA/a0lcbeeKAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-24T05:59:34.675367Z"},"content_sha256":"7b351e4619e4dd7456c223fa49740e0b882e57a2297c7d90e5cfc80850406d25","schema_version":"1.0","event_id":"sha256:7b351e4619e4dd7456c223fa49740e0b882e57a2297c7d90e5cfc80850406d25"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:OSFDBSGKGQQTCTQQ4I2CK26OAG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"\"My Answer is C\": First-Token Probabilities Do Not Match Text Answers in Instruction-Tuned Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Barbara Plank, Bolei Ma, Chengzhi Hu, Dirk Hovy, Frauke Kreuter, Leon Weber-Genzel, Paul R\\\"ottger, Xinpeng Wang","submitted_at":"2024-02-22T12:47:33Z","abstract_excerpt":"The open-ended nature of language generation makes the evaluation of autoregressive large language models (LLMs) challenging. One common evaluation approach uses multiple-choice questions (MCQ) to limit the response space. The model is then evaluated by ranking the candidate answers by the log probability of the first token prediction. However, first-tokens may not consistently reflect the final response output, due to model's diverse response styles such as starting with \"Sure\" or refusing to answer. Consequently, MCQ evaluation is not indicative of model behaviour when interacting with users"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.14499","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.14499/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/az9CtJbbyib84SuONudN6NZI/sRFFajjgEkxvAY82OHvbq4S0cagMS/Z/fXvaSK8DISMK1IwxTi0K8Ae8S2Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-24T05:59:34.675758Z"},"content_sha256":"31dccd5e55884c70f878a54de35ccf1f313c3080aa53689d04788267c8b929b0","schema_version":"1.0","event_id":"sha256:31dccd5e55884c70f878a54de35ccf1f313c3080aa53689d04788267c8b929b0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/bundle.json","state_url":"https://pith.science/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-24T05:59:34Z","links":{"resolver":"https://pith.science/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG","bundle":"https://pith.science/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/bundle.json","state":"https://pith.science/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OSFDBSGKGQQTCTQQ4I2CK26OAG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:OSFDBSGKGQQTCTQQ4I2CK26OAG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"17aaa71d5aab516edb5c61de8c67aa66ce95d25f6db8a02bcde3c62cc329fe8b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T12:47:33Z","title_canon_sha256":"b73bd8b535ef2e6b8a14fd6475a142229a719bbf27494853a4c2c50886004634"},"schema_version":"1.0","source":{"id":"2402.14499","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.14499","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"arxiv_version","alias_value":"2402.14499v2","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.14499","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_12","alias_value":"OSFDBSGKGQQT","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_16","alias_value":"OSFDBSGKGQQTCTQQ","created_at":"2026-07-05T08:40:13Z"},{"alias_kind":"pith_short_8","alias_value":"OSFDBSGK","created_at":"2026-07-05T08:40:13Z"}],"graph_snapshots":[{"event_id":"sha256:31dccd5e55884c70f878a54de35ccf1f313c3080aa53689d04788267c8b929b0","target":"graph","created_at":"2026-07-05T08:40:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.14499/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The open-ended nature of language generation makes the evaluation of autoregressive large language models (LLMs) challenging. One common evaluation approach uses multiple-choice questions (MCQ) to limit the response space. The model is then evaluated by ranking the candidate answers by the log probability of the first token prediction. However, first-tokens may not consistently reflect the final response output, due to model's diverse response styles such as starting with \"Sure\" or refusing to answer. Consequently, MCQ evaluation is not indicative of model behaviour when interacting with users","authors_text":"Barbara Plank, Bolei Ma, Chengzhi Hu, Dirk Hovy, Frauke Kreuter, Leon Weber-Genzel, Paul R\\\"ottger, Xinpeng Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T12:47:33Z","title":"\"My Answer is C\": First-Token Probabilities Do Not Match Text Answers in Instruction-Tuned Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.14499","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7b351e4619e4dd7456c223fa49740e0b882e57a2297c7d90e5cfc80850406d25","target":"record","created_at":"2026-07-05T08:40:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"17aaa71d5aab516edb5c61de8c67aa66ce95d25f6db8a02bcde3c62cc329fe8b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T12:47:33Z","title_canon_sha256":"b73bd8b535ef2e6b8a14fd6475a142229a719bbf27494853a4c2c50886004634"},"schema_version":"1.0","source":{"id":"2402.14499","kind":"arxiv","version":2}},"canonical_sha256":"748a30c8ca3421314e10e234256bce01ae294fdc42c46a0cc434ba38c610c563","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"748a30c8ca3421314e10e234256bce01ae294fdc42c46a0cc434ba38c610c563","first_computed_at":"2026-07-05T08:40:13.905075Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:40:13.905075Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NKJX38ODURDOwlkWUQsZ+QeuzUf1AaldbFciaHrgEbhdbZh66JrYcJkSXCvcUYFOxiGn6vLhEVrswhjAL0GuCA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:40:13.905723Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.14499","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7b351e4619e4dd7456c223fa49740e0b882e57a2297c7d90e5cfc80850406d25","sha256:31dccd5e55884c70f878a54de35ccf1f313c3080aa53689d04788267c8b929b0"],"state_sha256":"c5042b16d96b72d5dd5d633496d90be57d230432a7c98ea483ea993e147820eb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"R91Q+QJYNapP49drafrwodqprNeLCK+F+LU5eMlqKpTgur90f7dvtSjuug/yG4SPiQ63tnu2cXkQlXp1zWHtBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-24T05:59:34.677969Z","bundle_sha256":"1e94d073b33cd4fa9372d7965ceba84ca7042b7f1b3f1054f9a6104fac8126d2"}}