{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:ZF2SVLDDU2IN3TYZ5H5GZIS6UC","short_pith_number":"pith:ZF2SVLDD","schema_version":"1.0","canonical_sha256":"c9752aac63a690ddcf19e9fa6ca25ea09c90d79f57e48a5989f17118d78ce634","source":{"kind":"arxiv","id":"2412.10220","version":1},"attestation_state":"computed","paper":{"title":"How good is my story? Towards quantitative metrics for evaluating LLM-generated XAI narratives","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"David Martens, James Hinns, Timour Ichmoukhamedov","submitted_at":"2024-12-13T15:45:45Z","abstract_excerpt":"A rapidly developing application of LLMs in XAI is to convert quantitative explanations such as SHAP into user-friendly narratives to explain the decisions made by smaller prediction models. Evaluating the narratives without relying on human preference studies or surveys is becoming increasingly important in this field. In this work we propose a framework and explore several automated metrics to evaluate LLM-generated narratives for explanations of tabular classification tasks. We apply our approach to compare several state-of-the-art LLMs across different datasets and prompt types. As a demon"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.10220","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-13T15:45:45Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"3b126771baf22b83bfbe638b84c42c8445d4e144d83808b8a62bbf7db65a659a","abstract_canon_sha256":"d554e7c9ee201bcd2bd6ad539c303071e6cc96e2fe806171051d9b5526c949b9"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:48:48.876756Z","signature_b64":"NOXugC33WJRKZRaZ6oinY3AM5+Fio9pu34/bqImYtuC7FORuJJk81MCf7mAVt9tEef4rczun19LrrjXE5QVhDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c9752aac63a690ddcf19e9fa6ca25ea09c90d79f57e48a5989f17118d78ce634","last_reissued_at":"2026-07-05T09:48:48.876337Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:48:48.876337Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"How good is my story? Towards quantitative metrics for evaluating LLM-generated XAI narratives","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"David Martens, James Hinns, Timour Ichmoukhamedov","submitted_at":"2024-12-13T15:45:45Z","abstract_excerpt":"A rapidly developing application of LLMs in XAI is to convert quantitative explanations such as SHAP into user-friendly narratives to explain the decisions made by smaller prediction models. Evaluating the narratives without relying on human preference studies or surveys is becoming increasingly important in this field. In this work we propose a framework and explore several automated metrics to evaluate LLM-generated narratives for explanations of tabular classification tasks. We apply our approach to compare several state-of-the-art LLMs across different datasets and prompt types. As a demon"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.10220","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.10220/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.10220","created_at":"2026-07-05T09:48:48.876408+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.10220v1","created_at":"2026-07-05T09:48:48.876408+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.10220","created_at":"2026-07-05T09:48:48.876408+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZF2SVLDDU2IN","created_at":"2026-07-05T09:48:48.876408+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZF2SVLDDU2IN3TYZ","created_at":"2026-07-05T09:48:48.876408+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZF2SVLDD","created_at":"2026-07-05T09:48:48.876408+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.12543","citing_title":"A Two-Stage LLM Framework for Accessible and Verified XAI Explanations","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18311","citing_title":"On the Importance and Evaluation of Narrativity in Natural Language AI Explanations","ref_index":25,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC","json":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC.json","graph_json":"https://pith.science/api/pith-number/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/graph.json","events_json":"https://pith.science/api/pith-number/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/events.json","paper":"https://pith.science/paper/ZF2SVLDD"},"agent_actions":{"view_html":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC","download_json":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC.json","view_paper":"https://pith.science/paper/ZF2SVLDD","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.10220&json=true","fetch_graph":"https://pith.science/api/pith-number/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/graph.json","fetch_events":"https://pith.science/api/pith-number/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/action/storage_attestation","attest_author":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/action/author_attestation","sign_citation":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/action/citation_signature","submit_replication":"https://pith.science/pith/ZF2SVLDDU2IN3TYZ5H5GZIS6UC/action/replication_record"}},"created_at":"2026-07-05T09:48:48.876408+00:00","updated_at":"2026-07-05T09:48:48.876408+00:00"}