{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:N43N3Y2LZ6DDBL7WH5LHSA2KT5","short_pith_number":"pith:N43N3Y2L","schema_version":"1.0","canonical_sha256":"6f36dde34bcf8630aff63f5679034a9f5dd4791474ff0b763a7335dbee904941","source":{"kind":"arxiv","id":"2502.06207","version":3},"attestation_state":"computed","paper":{"title":"Is LLM an Overconfident Judge? Unveiling the Capabilities of LLMs in Detecting Offensive Language with Annotation Disagreement","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Bo Xu, Hongfei Lin, Junyu Lu, Kaichun Wang, Kai Ma, Kelaiti Xiao, Liang Yang, Roy Ka-Wei Lee","submitted_at":"2025-02-10T07:14:26Z","abstract_excerpt":"Large Language Models (LLMs) have become essential for offensive language detection, yet their ability to handle annotation disagreement remains underexplored. Disagreement samples, which arise from subjective interpretations, pose a unique challenge due to their ambiguous nature. Understanding how LLMs process these cases, particularly their confidence levels, can offer insight into their alignment with human annotators. This study systematically evaluates the performance of multiple LLMs in detecting offensive language at varying levels of annotation agreement. We analyze binary classificati"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.06207","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-10T07:14:26Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d97c2c27a31249dc9a3a0aab8b11a10b9db2f194517c14991cd5c80cc8ad9983","abstract_canon_sha256":"edc702cadd3d35f27d4ca35fee16fdd9aac1a8ffb5a100f4e974c23b145a4264"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:04:45.396400Z","signature_b64":"xDIwicREke92vabfhTuD2Zb+gdOQFBzdzZZJBhxnG+qXeV2GfOTWCAjru+7VnOKAFRYnurfWZK+My+wTxc6zDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6f36dde34bcf8630aff63f5679034a9f5dd4791474ff0b763a7335dbee904941","last_reissued_at":"2026-07-05T11:04:45.395813Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:04:45.395813Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Is LLM an Overconfident Judge? Unveiling the Capabilities of LLMs in Detecting Offensive Language with Annotation Disagreement","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Bo Xu, Hongfei Lin, Junyu Lu, Kaichun Wang, Kai Ma, Kelaiti Xiao, Liang Yang, Roy Ka-Wei Lee","submitted_at":"2025-02-10T07:14:26Z","abstract_excerpt":"Large Language Models (LLMs) have become essential for offensive language detection, yet their ability to handle annotation disagreement remains underexplored. Disagreement samples, which arise from subjective interpretations, pose a unique challenge due to their ambiguous nature. Understanding how LLMs process these cases, particularly their confidence levels, can offer insight into their alignment with human annotators. This study systematically evaluates the performance of multiple LLMs in detecting offensive language at varying levels of annotation agreement. We analyze binary classificati"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.06207","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.06207/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.06207","created_at":"2026-07-05T11:04:45.395883+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.06207v3","created_at":"2026-07-05T11:04:45.395883+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.06207","created_at":"2026-07-05T11:04:45.395883+00:00"},{"alias_kind":"pith_short_12","alias_value":"N43N3Y2LZ6DD","created_at":"2026-07-05T11:04:45.395883+00:00"},{"alias_kind":"pith_short_16","alias_value":"N43N3Y2LZ6DDBL7W","created_at":"2026-07-05T11:04:45.395883+00:00"},{"alias_kind":"pith_short_8","alias_value":"N43N3Y2L","created_at":"2026-07-05T11:04:45.395883+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.05329","citing_title":"Understanding Annotator Safety Policy with Interpretability","ref_index":36,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5","json":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5.json","graph_json":"https://pith.science/api/pith-number/N43N3Y2LZ6DDBL7WH5LHSA2KT5/graph.json","events_json":"https://pith.science/api/pith-number/N43N3Y2LZ6DDBL7WH5LHSA2KT5/events.json","paper":"https://pith.science/paper/N43N3Y2L"},"agent_actions":{"view_html":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5","download_json":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5.json","view_paper":"https://pith.science/paper/N43N3Y2L","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.06207&json=true","fetch_graph":"https://pith.science/api/pith-number/N43N3Y2LZ6DDBL7WH5LHSA2KT5/graph.json","fetch_events":"https://pith.science/api/pith-number/N43N3Y2LZ6DDBL7WH5LHSA2KT5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5/action/storage_attestation","attest_author":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5/action/author_attestation","sign_citation":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5/action/citation_signature","submit_replication":"https://pith.science/pith/N43N3Y2LZ6DDBL7WH5LHSA2KT5/action/replication_record"}},"created_at":"2026-07-05T11:04:45.395883+00:00","updated_at":"2026-07-05T11:04:45.395883+00:00"}