{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:CMTADOIODTHFI4XHQRU35DAEDF","short_pith_number":"pith:CMTADOIO","canonical_record":{"source":{"id":"2507.02856","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-03T17:59:02Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"e484311a96b94df1e142d2fa6299c7e3db7a48ba4f8916fcadd294968e161d1f","abstract_canon_sha256":"0dbc0c111c2d2f9f417aeb73bd53d3bf99b10b87ed364ba6348c166b647695c3"},"schema_version":"1.0"},"canonical_sha256":"132601b90e1cce5472e78469be8c04194bc0b4de91ec2703c10af7671460034b","source":{"kind":"arxiv","id":"2507.02856","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.02856","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"arxiv_version","alias_value":"2507.02856v1","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.02856","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_12","alias_value":"CMTADOIODTHF","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_16","alias_value":"CMTADOIODTHFI4XH","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_8","alias_value":"CMTADOIO","created_at":"2026-07-05T11:31:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:CMTADOIODTHFI4XHQRU35DAEDF","target":"record","payload":{"canonical_record":{"source":{"id":"2507.02856","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-03T17:59:02Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"e484311a96b94df1e142d2fa6299c7e3db7a48ba4f8916fcadd294968e161d1f","abstract_canon_sha256":"0dbc0c111c2d2f9f417aeb73bd53d3bf99b10b87ed364ba6348c166b647695c3"},"schema_version":"1.0"},"canonical_sha256":"132601b90e1cce5472e78469be8c04194bc0b4de91ec2703c10af7671460034b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:31:35.401030Z","signature_b64":"jcTCpT2LMyeUUFdgmanvYgcUdgXM1V5F936xUx5GmaXO3+Q3vlHM/Lqy5gjlZJT0Ma4aIcMgnak14L/q8vT5Bw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"132601b90e1cce5472e78469be8c04194bc0b4de91ec2703c10af7671460034b","last_reissued_at":"2026-07-05T11:31:35.400545Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:31:35.400545Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.02856","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:31:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OhqM/iYDH6aeNtgXXY762YvFo9T6/2c5URrrlYtpO09CNHTSUpyRjW8PnmJ5VDWilZsaLWvbKj1MTTp1P8xQDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T04:16:17.370031Z"},"content_sha256":"36bbec547eeba1e1df4a9e189f1156095797bfd0c123a3b2597281877a047fad","schema_version":"1.0","event_id":"sha256:36bbec547eeba1e1df4a9e189f1156095797bfd0c123a3b2597281877a047fad"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:CMTADOIODTHFI4XHQRU35DAEDF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Answer Matching Outperforms Multiple Choice for Language Model Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ameya Prabhu, Jonas Geiping, Moritz Hardt, Nikhil Chandak, Shashwat Goel","submitted_at":"2025-07-03T17:59:02Z","abstract_excerpt":"Multiple choice benchmarks have long been the workhorse of language model evaluation because grading multiple choice is objective and easy to automate. However, we show multiple choice questions from popular benchmarks can often be answered without even seeing the question. These shortcuts arise from a fundamental limitation of discriminative evaluation not shared by evaluations of the model's free-form, generative answers. Until recently, there appeared to be no viable, scalable alternative to multiple choice--but, we show that this has changed. We consider generative evaluation via what we c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.02856","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.02856/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:31:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wb+9yqbs1rPjOrXQpMcTo8eokSuvKRu60qaI4sL10pyVGGMWs12AyxPLxtLoAL0qrML36u/ACVZujS+3j2t2CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T04:16:17.370832Z"},"content_sha256":"26d1641ab09fedea38288904adc221a4f2c74d0492a5405777402448ad23ac24","schema_version":"1.0","event_id":"sha256:26d1641ab09fedea38288904adc221a4f2c74d0492a5405777402448ad23ac24"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CMTADOIODTHFI4XHQRU35DAEDF/bundle.json","state_url":"https://pith.science/pith/CMTADOIODTHFI4XHQRU35DAEDF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CMTADOIODTHFI4XHQRU35DAEDF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T04:16:17Z","links":{"resolver":"https://pith.science/pith/CMTADOIODTHFI4XHQRU35DAEDF","bundle":"https://pith.science/pith/CMTADOIODTHFI4XHQRU35DAEDF/bundle.json","state":"https://pith.science/pith/CMTADOIODTHFI4XHQRU35DAEDF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CMTADOIODTHFI4XHQRU35DAEDF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:CMTADOIODTHFI4XHQRU35DAEDF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0dbc0c111c2d2f9f417aeb73bd53d3bf99b10b87ed364ba6348c166b647695c3","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-03T17:59:02Z","title_canon_sha256":"e484311a96b94df1e142d2fa6299c7e3db7a48ba4f8916fcadd294968e161d1f"},"schema_version":"1.0","source":{"id":"2507.02856","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.02856","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"arxiv_version","alias_value":"2507.02856v1","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.02856","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_12","alias_value":"CMTADOIODTHF","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_16","alias_value":"CMTADOIODTHFI4XH","created_at":"2026-07-05T11:31:35Z"},{"alias_kind":"pith_short_8","alias_value":"CMTADOIO","created_at":"2026-07-05T11:31:35Z"}],"graph_snapshots":[{"event_id":"sha256:26d1641ab09fedea38288904adc221a4f2c74d0492a5405777402448ad23ac24","target":"graph","created_at":"2026-07-05T11:31:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.02856/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multiple choice benchmarks have long been the workhorse of language model evaluation because grading multiple choice is objective and easy to automate. However, we show multiple choice questions from popular benchmarks can often be answered without even seeing the question. These shortcuts arise from a fundamental limitation of discriminative evaluation not shared by evaluations of the model's free-form, generative answers. Until recently, there appeared to be no viable, scalable alternative to multiple choice--but, we show that this has changed. We consider generative evaluation via what we c","authors_text":"Ameya Prabhu, Jonas Geiping, Moritz Hardt, Nikhil Chandak, Shashwat Goel","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-03T17:59:02Z","title":"Answer Matching Outperforms Multiple Choice for Language Model Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.02856","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:36bbec547eeba1e1df4a9e189f1156095797bfd0c123a3b2597281877a047fad","target":"record","created_at":"2026-07-05T11:31:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0dbc0c111c2d2f9f417aeb73bd53d3bf99b10b87ed364ba6348c166b647695c3","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-03T17:59:02Z","title_canon_sha256":"e484311a96b94df1e142d2fa6299c7e3db7a48ba4f8916fcadd294968e161d1f"},"schema_version":"1.0","source":{"id":"2507.02856","kind":"arxiv","version":1}},"canonical_sha256":"132601b90e1cce5472e78469be8c04194bc0b4de91ec2703c10af7671460034b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"132601b90e1cce5472e78469be8c04194bc0b4de91ec2703c10af7671460034b","first_computed_at":"2026-07-05T11:31:35.400545Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:31:35.400545Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jcTCpT2LMyeUUFdgmanvYgcUdgXM1V5F936xUx5GmaXO3+Q3vlHM/Lqy5gjlZJT0Ma4aIcMgnak14L/q8vT5Bw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:31:35.401030Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.02856","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:36bbec547eeba1e1df4a9e189f1156095797bfd0c123a3b2597281877a047fad","sha256:26d1641ab09fedea38288904adc221a4f2c74d0492a5405777402448ad23ac24"],"state_sha256":"686835de56bd2a4208db1ea76f4ff4105ad3514286b0495dd6871beaea889629"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z/Hil7tHYY3RWTa4+8byYGI9UFfHZVjYVsjCoEL+vf+TGdCf0/q1kMsKFiIjxrxRPaw6CE1MW2E9kIjukRZBDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T04:16:17.376829Z","bundle_sha256":"ec083283758f1690b66d21e2342758d8034be69388c5a43d3baf1b82b2a38469"}}