{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:XO6D2ITOVFGOX4CPV4SM2KXFXC","short_pith_number":"pith:XO6D2ITO","schema_version":"1.0","canonical_sha256":"bbbc3d226ea94cebf04faf24cd2ae5b895bbd3698f0d0fe8910bd6abfed6c668","source":{"kind":"arxiv","id":"2403.12675","version":2},"attestation_state":"computed","paper":{"title":"Pragmatic Competence Evaluation of Large Language Models for the Korean Language","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dojun Park, Hyeyun Jeong, Jiwoo Lee, Seohyun Park, Sungeun Lee","submitted_at":"2024-03-19T12:21:20Z","abstract_excerpt":"Benchmarks play a significant role in the current evaluation of Large Language Models (LLMs), yet they often overlook the models' abilities to capture the nuances of human language, primarily focusing on evaluating embedded knowledge and technical skills. To address this gap, our study evaluates how well LLMs understand context-dependent expressions from a pragmatic standpoint, specifically in Korean. We use both Multiple-Choice Questions (MCQs) for automatic evaluation and Open-Ended Questions (OEQs) assessed by human experts. Our results show that GPT-4 leads with scores of 81.11 in MCQs and"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2403.12675","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-19T12:21:20Z","cross_cats_sorted":[],"title_canon_sha256":"9799673034935897e82abcd02f8dd6c336ab255811703cf3afc0b29047fc63fb","abstract_canon_sha256":"4317a05ad2a560183f90276db1bfaecba8e649196d7e8df7d56f15c3f55fe168"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:42.202141Z","signature_b64":"QnZDZKTlDvd6qvVBB3kDiwSvFsITPnWed5xXqVg1HUizwKLz+WJEh7uMQ/ARd3UYTPC+Mon7BwV2DpJPhqf6Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bbbc3d226ea94cebf04faf24cd2ae5b895bbd3698f0d0fe8910bd6abfed6c668","last_reissued_at":"2026-07-05T09:21:42.201708Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:42.201708Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Pragmatic Competence Evaluation of Large Language Models for the Korean Language","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dojun Park, Hyeyun Jeong, Jiwoo Lee, Seohyun Park, Sungeun Lee","submitted_at":"2024-03-19T12:21:20Z","abstract_excerpt":"Benchmarks play a significant role in the current evaluation of Large Language Models (LLMs), yet they often overlook the models' abilities to capture the nuances of human language, primarily focusing on evaluating embedded knowledge and technical skills. To address this gap, our study evaluates how well LLMs understand context-dependent expressions from a pragmatic standpoint, specifically in Korean. We use both Multiple-Choice Questions (MCQs) for automatic evaluation and Open-Ended Questions (OEQs) assessed by human experts. Our results show that GPT-4 leads with scores of 81.11 in MCQs and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.12675","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.12675/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2403.12675","created_at":"2026-07-05T09:21:42.201764+00:00"},{"alias_kind":"arxiv_version","alias_value":"2403.12675v2","created_at":"2026-07-05T09:21:42.201764+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.12675","created_at":"2026-07-05T09:21:42.201764+00:00"},{"alias_kind":"pith_short_12","alias_value":"XO6D2ITOVFGO","created_at":"2026-07-05T09:21:42.201764+00:00"},{"alias_kind":"pith_short_16","alias_value":"XO6D2ITOVFGOX4CP","created_at":"2026-07-05T09:21:42.201764+00:00"},{"alias_kind":"pith_short_8","alias_value":"XO6D2ITO","created_at":"2026-07-05T09:21:42.201764+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2507.03378","citing_title":"Making Sense of Korean Sentences: A Comprehensive Evaluation of LLMs through KoSEnd Dataset","ref_index":40,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC","json":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC.json","graph_json":"https://pith.science/api/pith-number/XO6D2ITOVFGOX4CPV4SM2KXFXC/graph.json","events_json":"https://pith.science/api/pith-number/XO6D2ITOVFGOX4CPV4SM2KXFXC/events.json","paper":"https://pith.science/paper/XO6D2ITO"},"agent_actions":{"view_html":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC","download_json":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC.json","view_paper":"https://pith.science/paper/XO6D2ITO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2403.12675&json=true","fetch_graph":"https://pith.science/api/pith-number/XO6D2ITOVFGOX4CPV4SM2KXFXC/graph.json","fetch_events":"https://pith.science/api/pith-number/XO6D2ITOVFGOX4CPV4SM2KXFXC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC/action/storage_attestation","attest_author":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC/action/author_attestation","sign_citation":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC/action/citation_signature","submit_replication":"https://pith.science/pith/XO6D2ITOVFGOX4CPV4SM2KXFXC/action/replication_record"}},"created_at":"2026-07-05T09:21:42.201764+00:00","updated_at":"2026-07-05T09:21:42.201764+00:00"}