{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:654SBB7SLKWPMSVJTE7QMKYBZD","short_pith_number":"pith:654SBB7S","schema_version":"1.0","canonical_sha256":"f7792087f25aacf64aa9993f062b01c8fd3db0a2ecffcfa4c63ad674470e39c5","source":{"kind":"arxiv","id":"2306.09296","version":3},"attestation_state":"computed","paper":{"title":"KoLA: Carefully Benchmarking World Knowledge of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Amy Xin, Bin Xu, Chunyang Li, Daniel Zhang-Li, Hailong Jin, Hanming Li, Hao Peng, Jianhui Chen, Jie Tang, Jifan Yu, Jinxin Liu, Ji Qi, Juanzi Li, Kaifeng Yun, Kaisheng Zeng, Lei Hou, Linlu Gong, Nianyi Lin, Ning Ding, Shangqing Tu, Shulin Cao, Weikai Li, Xiaohan Zhang, Xiaozhi Wang, Xin Lv, Yantao Liu, Yong Guan, Yuan Yao, Yu Gu, Yunjia Qi, Yushi Bai, Zheyuan Zhang, Zhili Wu, Zhiyuan Liu, Zijun Yao","submitted_at":"2023-06-15T17:20:46Z","abstract_excerpt":"The unprecedented performance of large language models (LLMs) necessitates improvements in evaluations. Rather than merely exploring the breadth of LLM abilities, we believe meticulous and thoughtful designs are essential to thorough, unbiased, and applicable evaluations. Given the importance of world knowledge to LLMs, we construct a Knowledge-oriented LLM Assessment benchmark (KoLA), in which we carefully design three crucial factors: (1) For \\textbf{ability modeling}, we mimic human cognition to form a four-level taxonomy of knowledge-related abilities, covering $19$ tasks. (2) For \\textbf{"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2306.09296","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-06-15T17:20:46Z","cross_cats_sorted":[],"title_canon_sha256":"8193e4b7b95963e4195edab98b71ebc73617a26f675fa65f0bc22c55ba086155","abstract_canon_sha256":"76ab1b7f3dbc30a038bb46b364698b2cd11d9dee48b1fd06843266371844eac7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:38:02.377242Z","signature_b64":"kjZbIakaNUlBTLXV0Vbdn9Up7gVUslABf144p4Rk0FkCxz0U7ixl5046YPpzU0NzabjhNlvuFKnitUOxxULBDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f7792087f25aacf64aa9993f062b01c8fd3db0a2ecffcfa4c63ad674470e39c5","last_reissued_at":"2026-07-05T08:38:02.376798Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:38:02.376798Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"KoLA: Carefully Benchmarking World Knowledge of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Amy Xin, Bin Xu, Chunyang Li, Daniel Zhang-Li, Hailong Jin, Hanming Li, Hao Peng, Jianhui Chen, Jie Tang, Jifan Yu, Jinxin Liu, Ji Qi, Juanzi Li, Kaifeng Yun, Kaisheng Zeng, Lei Hou, Linlu Gong, Nianyi Lin, Ning Ding, Shangqing Tu, Shulin Cao, Weikai Li, Xiaohan Zhang, Xiaozhi Wang, Xin Lv, Yantao Liu, Yong Guan, Yuan Yao, Yu Gu, Yunjia Qi, Yushi Bai, Zheyuan Zhang, Zhili Wu, Zhiyuan Liu, Zijun Yao","submitted_at":"2023-06-15T17:20:46Z","abstract_excerpt":"The unprecedented performance of large language models (LLMs) necessitates improvements in evaluations. Rather than merely exploring the breadth of LLM abilities, we believe meticulous and thoughtful designs are essential to thorough, unbiased, and applicable evaluations. Given the importance of world knowledge to LLMs, we construct a Knowledge-oriented LLM Assessment benchmark (KoLA), in which we carefully design three crucial factors: (1) For \\textbf{ability modeling}, we mimic human cognition to form a four-level taxonomy of knowledge-related abilities, covering $19$ tasks. (2) For \\textbf{"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.09296","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.09296/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2306.09296","created_at":"2026-07-05T08:38:02.376857+00:00"},{"alias_kind":"arxiv_version","alias_value":"2306.09296v3","created_at":"2026-07-05T08:38:02.376857+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.09296","created_at":"2026-07-05T08:38:02.376857+00:00"},{"alias_kind":"pith_short_12","alias_value":"654SBB7SLKWP","created_at":"2026-07-05T08:38:02.376857+00:00"},{"alias_kind":"pith_short_16","alias_value":"654SBB7SLKWPMSVJ","created_at":"2026-07-05T08:38:02.376857+00:00"},{"alias_kind":"pith_short_8","alias_value":"654SBB7S","created_at":"2026-07-05T08:38:02.376857+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.01120","citing_title":"Diagnosing LLM Arbitration Behavior over Pre-evidence Epistemic States in RAG-based Fact-Checking","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18771","citing_title":"LWGR: Lagrangian-Constrained Personalized World Knowledge for Generative Recommendation","ref_index":57,"is_internal_anchor":false},{"citing_arxiv_id":"2401.03568","citing_title":"Agent AI: Surveying the Horizons of Multimodal Interaction","ref_index":210,"is_internal_anchor":false},{"citing_arxiv_id":"2503.07265","citing_title":"WISE: A World Knowledge-Informed Semantic Evaluation for Text-to-Image Generation","ref_index":57,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08964","citing_title":"Trustworthy AI: Ensuring Reliability and Accountability from Models to Agents","ref_index":166,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD","json":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD.json","graph_json":"https://pith.science/api/pith-number/654SBB7SLKWPMSVJTE7QMKYBZD/graph.json","events_json":"https://pith.science/api/pith-number/654SBB7SLKWPMSVJTE7QMKYBZD/events.json","paper":"https://pith.science/paper/654SBB7S"},"agent_actions":{"view_html":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD","download_json":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD.json","view_paper":"https://pith.science/paper/654SBB7S","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2306.09296&json=true","fetch_graph":"https://pith.science/api/pith-number/654SBB7SLKWPMSVJTE7QMKYBZD/graph.json","fetch_events":"https://pith.science/api/pith-number/654SBB7SLKWPMSVJTE7QMKYBZD/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD/action/timestamp_anchor","attest_storage":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD/action/storage_attestation","attest_author":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD/action/author_attestation","sign_citation":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD/action/citation_signature","submit_replication":"https://pith.science/pith/654SBB7SLKWPMSVJTE7QMKYBZD/action/replication_record"}},"created_at":"2026-07-05T08:38:02.376857+00:00","updated_at":"2026-07-05T08:38:02.376857+00:00"}