{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:PNYBHG7ZHOMH3CO2E2U4ZFC275","short_pith_number":"pith:PNYBHG7Z","canonical_record":{"source":{"id":"2310.14703","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-23T08:45:12Z","cross_cats_sorted":[],"title_canon_sha256":"b3a010dc7a1908049da14e53bcbfb34d0597f2e8f55b9074942a93b726065f07","abstract_canon_sha256":"bdccc260210df72ed72a72d2ba8700f88658c148ad59dc11e78b32fe6f5f26d5"},"schema_version":"1.0"},"canonical_sha256":"7b70139bf93b987d89da26a9cc945aff4177eef5d48d2df7da29fad523c25784","source":{"kind":"arxiv","id":"2310.14703","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.14703","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"arxiv_version","alias_value":"2310.14703v2","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.14703","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_12","alias_value":"PNYBHG7ZHOMH","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_16","alias_value":"PNYBHG7ZHOMH3CO2","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_8","alias_value":"PNYBHG7Z","created_at":"2026-07-05T07:38:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:PNYBHG7ZHOMH3CO2E2U4ZFC275","target":"record","payload":{"canonical_record":{"source":{"id":"2310.14703","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-23T08:45:12Z","cross_cats_sorted":[],"title_canon_sha256":"b3a010dc7a1908049da14e53bcbfb34d0597f2e8f55b9074942a93b726065f07","abstract_canon_sha256":"bdccc260210df72ed72a72d2ba8700f88658c148ad59dc11e78b32fe6f5f26d5"},"schema_version":"1.0"},"canonical_sha256":"7b70139bf93b987d89da26a9cc945aff4177eef5d48d2df7da29fad523c25784","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:38:31.981974Z","signature_b64":"9h1IyTUH9AfulMb2O9gXJVDKEK5/SrsFRYbQ26Vz4Mu+3xc+d97GJMnoAoyi43kJ11IE7/8XVhIDy69maxL+Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7b70139bf93b987d89da26a9cc945aff4177eef5d48d2df7da29fad523c25784","last_reissued_at":"2026-07-05T07:38:31.981586Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:38:31.981586Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.14703","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VxwbYs/W4esSXvPPLoOElcebgRsulTtVdL1LAvETSqCIl8pwzI5JmBav/yiKKZE1YAF5gneuPgJmW9KJVdJ7Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T02:08:37.822893Z"},"content_sha256":"5ffd880a5df4f906e323297cde33ec5a8996744458adc78b478715baaef29822","schema_version":"1.0","event_id":"sha256:5ffd880a5df4f906e323297cde33ec5a8996744458adc78b478715baaef29822"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:PNYBHG7ZHOMH3CO2E2U4ZFC275","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Establishing Vocabulary Tests as a Benchmark for Evaluating Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Beatriz Berm\\'udez-Margaretto, Elena Merino-G\\'omez, Gonzalo Mart\\'inez, Javier Conde, Jos\\'e Alberto Hern\\'andez, Marc Brysbaert, Pedro Reviriego","submitted_at":"2023-10-23T08:45:12Z","abstract_excerpt":"Vocabulary tests, once a cornerstone of language modeling evaluation, have been largely overlooked in the current landscape of Large Language Models (LLMs) like Llama, Mistral, and GPT. While most LLM evaluation benchmarks focus on specific tasks or domain-specific knowledge, they often neglect the fundamental linguistic aspects of language understanding and production. In this paper, we advocate for the revival of vocabulary tests as a valuable tool for assessing LLM performance. We evaluate seven LLMs using two vocabulary test formats across two languages and uncover surprising gaps in their"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.14703","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.14703/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:38:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"16YoCK7Vt3jHe7A6CBc8BjUKvH8IneV8Foj0t9+NxJL6cKqLYQwoLvZGmYPxdwqSLulhI5DM4cMLxBHDiwGjCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T02:08:37.823417Z"},"content_sha256":"4538cecaa6d71dcb38645cbbdd4131c7755b166a4ad4615c2dea4c8e916321d0","schema_version":"1.0","event_id":"sha256:4538cecaa6d71dcb38645cbbdd4131c7755b166a4ad4615c2dea4c8e916321d0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/bundle.json","state_url":"https://pith.science/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-23T02:08:37Z","links":{"resolver":"https://pith.science/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275","bundle":"https://pith.science/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/bundle.json","state":"https://pith.science/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PNYBHG7ZHOMH3CO2E2U4ZFC275/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:PNYBHG7ZHOMH3CO2E2U4ZFC275","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bdccc260210df72ed72a72d2ba8700f88658c148ad59dc11e78b32fe6f5f26d5","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-23T08:45:12Z","title_canon_sha256":"b3a010dc7a1908049da14e53bcbfb34d0597f2e8f55b9074942a93b726065f07"},"schema_version":"1.0","source":{"id":"2310.14703","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.14703","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"arxiv_version","alias_value":"2310.14703v2","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.14703","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_12","alias_value":"PNYBHG7ZHOMH","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_16","alias_value":"PNYBHG7ZHOMH3CO2","created_at":"2026-07-05T07:38:31Z"},{"alias_kind":"pith_short_8","alias_value":"PNYBHG7Z","created_at":"2026-07-05T07:38:31Z"}],"graph_snapshots":[{"event_id":"sha256:4538cecaa6d71dcb38645cbbdd4131c7755b166a4ad4615c2dea4c8e916321d0","target":"graph","created_at":"2026-07-05T07:38:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.14703/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Vocabulary tests, once a cornerstone of language modeling evaluation, have been largely overlooked in the current landscape of Large Language Models (LLMs) like Llama, Mistral, and GPT. While most LLM evaluation benchmarks focus on specific tasks or domain-specific knowledge, they often neglect the fundamental linguistic aspects of language understanding and production. In this paper, we advocate for the revival of vocabulary tests as a valuable tool for assessing LLM performance. We evaluate seven LLMs using two vocabulary test formats across two languages and uncover surprising gaps in their","authors_text":"Beatriz Berm\\'udez-Margaretto, Elena Merino-G\\'omez, Gonzalo Mart\\'inez, Javier Conde, Jos\\'e Alberto Hern\\'andez, Marc Brysbaert, Pedro Reviriego","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-23T08:45:12Z","title":"Establishing Vocabulary Tests as a Benchmark for Evaluating Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.14703","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5ffd880a5df4f906e323297cde33ec5a8996744458adc78b478715baaef29822","target":"record","created_at":"2026-07-05T07:38:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bdccc260210df72ed72a72d2ba8700f88658c148ad59dc11e78b32fe6f5f26d5","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-23T08:45:12Z","title_canon_sha256":"b3a010dc7a1908049da14e53bcbfb34d0597f2e8f55b9074942a93b726065f07"},"schema_version":"1.0","source":{"id":"2310.14703","kind":"arxiv","version":2}},"canonical_sha256":"7b70139bf93b987d89da26a9cc945aff4177eef5d48d2df7da29fad523c25784","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7b70139bf93b987d89da26a9cc945aff4177eef5d48d2df7da29fad523c25784","first_computed_at":"2026-07-05T07:38:31.981586Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:38:31.981586Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9h1IyTUH9AfulMb2O9gXJVDKEK5/SrsFRYbQ26Vz4Mu+3xc+d97GJMnoAoyi43kJ11IE7/8XVhIDy69maxL+Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:38:31.981974Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.14703","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5ffd880a5df4f906e323297cde33ec5a8996744458adc78b478715baaef29822","sha256:4538cecaa6d71dcb38645cbbdd4131c7755b166a4ad4615c2dea4c8e916321d0"],"state_sha256":"0957f5f4249238cd813f6a63b57837ec6a754a320a45a6fa60f5698af0553631"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dz7HRaRZPWJUzPlhafD6Hcuq6q6g03ez89AUzIi09P2sB3GaeVI8ftHdj3vKLH/j0WMgv9iNIAV8w+65DTPDAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-23T02:08:37.828301Z","bundle_sha256":"bbcb81bf1a9c033af94b63aa442fa6afc9617ceb8d36c0dd6eff9694ffb015df"}}