{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:VWO2OWZ6J3VJSVSSVPILKNVDNM","short_pith_number":"pith:VWO2OWZ6","schema_version":"1.0","canonical_sha256":"ad9da75b3e4eea995652abd0b536a36b39c21b81609f3e0c5637fa81e0b6f728","source":{"kind":"arxiv","id":"2510.01782","version":2},"attestation_state":"computed","paper":{"title":"Can LLMs Refuse Questions They Do Not Know? Measuring Knowledge-Aware Refusal in Factual Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Haining Yu, Jie Xu, Junhao Dong, Libo Qin, Qiguang Chen, Wenbo Pan, Xiaohua Jia, Xinfeng Li","submitted_at":"2025-10-02T08:20:36Z","abstract_excerpt":"Large Language Models (LLMs) should refuse to answer questions beyond their knowledge. This capability, which we term knowledge-aware refusal, is crucial for factual reliability, while existing metrics fail to capture this ability. In this work, we propose the Refusal Index (RI), a novel and principled metric that measures how accurately LLMs refuse questions they do not know. We define RI as Spearman's rank correlation between refusal probability and error probability. RI is practically measurable with a lightweight two-pass evaluation method which only require observed refusal rates across t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2510.01782","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-10-02T08:20:36Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c8bb22d4e74afc591f16e1029b14a9bbec0f5a7a944a0b094179353a6aad3836","abstract_canon_sha256":"571bea34fef0fc58a4068c9be1bcf195299ed3465e27c400f6244bd2e80882a0"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-20T00:02:56.177848Z","signature_b64":"kkmGPCLQXD1BcfkOIBZKYXmJHeb7IolzcKHXV6NvVUBb2DK8exNUXzfgQhQTaDA2PUBN7tsTkUjwUa4qZ8tECA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ad9da75b3e4eea995652abd0b536a36b39c21b81609f3e0c5637fa81e0b6f728","last_reissued_at":"2026-05-20T00:02:56.176905Z","signature_status":"signed_v1","first_computed_at":"2026-05-20T00:02:56.176905Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Can LLMs Refuse Questions They Do Not Know? Measuring Knowledge-Aware Refusal in Factual Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Haining Yu, Jie Xu, Junhao Dong, Libo Qin, Qiguang Chen, Wenbo Pan, Xiaohua Jia, Xinfeng Li","submitted_at":"2025-10-02T08:20:36Z","abstract_excerpt":"Large Language Models (LLMs) should refuse to answer questions beyond their knowledge. This capability, which we term knowledge-aware refusal, is crucial for factual reliability, while existing metrics fail to capture this ability. In this work, we propose the Refusal Index (RI), a novel and principled metric that measures how accurately LLMs refuse questions they do not know. We define RI as Spearman's rank correlation between refusal probability and error probability. RI is practically measurable with a lightweight two-pass evaluation method which only require observed refusal rates across t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2510.01782","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2510.01782/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2510.01782","created_at":"2026-05-20T00:02:56.177075+00:00"},{"alias_kind":"arxiv_version","alias_value":"2510.01782v2","created_at":"2026-05-20T00:02:56.177075+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2510.01782","created_at":"2026-05-20T00:02:56.177075+00:00"},{"alias_kind":"pith_short_12","alias_value":"VWO2OWZ6J3VJ","created_at":"2026-05-20T00:02:56.177075+00:00"},{"alias_kind":"pith_short_16","alias_value":"VWO2OWZ6J3VJSVSS","created_at":"2026-05-20T00:02:56.177075+00:00"},{"alias_kind":"pith_short_8","alias_value":"VWO2OWZ6","created_at":"2026-05-20T00:02:56.177075+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM","json":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM.json","graph_json":"https://pith.science/api/pith-number/VWO2OWZ6J3VJSVSSVPILKNVDNM/graph.json","events_json":"https://pith.science/api/pith-number/VWO2OWZ6J3VJSVSSVPILKNVDNM/events.json","paper":"https://pith.science/paper/VWO2OWZ6"},"agent_actions":{"view_html":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM","download_json":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM.json","view_paper":"https://pith.science/paper/VWO2OWZ6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2510.01782&json=true","fetch_graph":"https://pith.science/api/pith-number/VWO2OWZ6J3VJSVSSVPILKNVDNM/graph.json","fetch_events":"https://pith.science/api/pith-number/VWO2OWZ6J3VJSVSSVPILKNVDNM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM/action/storage_attestation","attest_author":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM/action/author_attestation","sign_citation":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM/action/citation_signature","submit_replication":"https://pith.science/pith/VWO2OWZ6J3VJSVSSVPILKNVDNM/action/replication_record"}},"created_at":"2026-05-20T00:02:56.177075+00:00","updated_at":"2026-05-20T00:02:56.177075+00:00"}