{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:4HWBXJ2Q2A6DLL7OL4FHODXRXT","short_pith_number":"pith:4HWBXJ2Q","canonical_record":{"source":{"id":"2311.03533","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-06T21:14:40Z","cross_cats_sorted":[],"title_canon_sha256":"49e50d297ac680beba66fc50f755789cf17409e5ace361d3cc14faac64ce33c5","abstract_canon_sha256":"fd183c1db094f111573892b243458cc8b94e8efc72479e20919c7a360732aac8"},"schema_version":"1.0"},"canonical_sha256":"e1ec1ba750d03c35afee5f0a770ef1bcf3a78b56c246193c30708a0951b4e932","source":{"kind":"arxiv","id":"2311.03533","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.03533","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"arxiv_version","alias_value":"2311.03533v1","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.03533","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_12","alias_value":"4HWBXJ2Q2A6D","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_16","alias_value":"4HWBXJ2Q2A6DLL7O","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_8","alias_value":"4HWBXJ2Q","created_at":"2026-07-05T07:09:59Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:4HWBXJ2Q2A6DLL7OL4FHODXRXT","target":"record","payload":{"canonical_record":{"source":{"id":"2311.03533","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-06T21:14:40Z","cross_cats_sorted":[],"title_canon_sha256":"49e50d297ac680beba66fc50f755789cf17409e5ace361d3cc14faac64ce33c5","abstract_canon_sha256":"fd183c1db094f111573892b243458cc8b94e8efc72479e20919c7a360732aac8"},"schema_version":"1.0"},"canonical_sha256":"e1ec1ba750d03c35afee5f0a770ef1bcf3a78b56c246193c30708a0951b4e932","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:09:59.759499Z","signature_b64":"t4E1bthd2+4lSYh6mebgV+Xm4LEZRTAB/PVp8vx4anq/65+i8bMffQVoxzENbQhTiXcLMJQhCrKYka9IW4jHDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e1ec1ba750d03c35afee5f0a770ef1bcf3a78b56c246193c30708a0951b4e932","last_reissued_at":"2026-07-05T07:09:59.758853Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:09:59.758853Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.03533","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:09:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"p3puuSNoxLy5RzhWuS/E/840fbNa9L13oYBj13yLpMhxgt+Yqeps7xuV5igwgVMifvmwv25G2BII14H4R5jrDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:18:12.244252Z"},"content_sha256":"cb6e7b7313ffa14ed89bccb4ec2ce06dfb101b680ef962cfa6a33e5db963c31a","schema_version":"1.0","event_id":"sha256:cb6e7b7313ffa14ed89bccb4ec2ce06dfb101b680ef962cfa6a33e5db963c31a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:4HWBXJ2Q2A6DLL7OL4FHODXRXT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Quantifying Uncertainty in Natural Language Explanations of Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chirag Agarwal, Himabindu Lakkaraju, Sree Harsha Tanneru","submitted_at":"2023-11-06T21:14:40Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly used as powerful tools for several high-stakes natural language processing (NLP) applications. Recent prompting works claim to elicit intermediate reasoning steps and key tokens that serve as proxy explanations for LLM predictions. However, there is no certainty whether these explanations are reliable and reflect the LLMs behavior. In this work, we make one of the first attempts at quantifying the uncertainty in explanations of LLMs. To this end, we propose two novel metrics -- $\\textit{Verbalized Uncertainty}$ and $\\textit{Probing Uncertainty}$ --"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.03533","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.03533/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:09:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"brZn5Z15++UEEGmJoZzMHZA5JHjUnhVGxM0j9QfLxvvz9d/KvAjjlq519bTPvZOleLI+Va3KqhP+OeH2ZlB6DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:18:12.244738Z"},"content_sha256":"880a23ffefda293762bbdc1a2608a508794b5eec88f87f6dd7ab59b6ca1c8ff7","schema_version":"1.0","event_id":"sha256:880a23ffefda293762bbdc1a2608a508794b5eec88f87f6dd7ab59b6ca1c8ff7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/bundle.json","state_url":"https://pith.science/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T07:18:12Z","links":{"resolver":"https://pith.science/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT","bundle":"https://pith.science/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/bundle.json","state":"https://pith.science/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4HWBXJ2Q2A6DLL7OL4FHODXRXT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:4HWBXJ2Q2A6DLL7OL4FHODXRXT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fd183c1db094f111573892b243458cc8b94e8efc72479e20919c7a360732aac8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-06T21:14:40Z","title_canon_sha256":"49e50d297ac680beba66fc50f755789cf17409e5ace361d3cc14faac64ce33c5"},"schema_version":"1.0","source":{"id":"2311.03533","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.03533","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"arxiv_version","alias_value":"2311.03533v1","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.03533","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_12","alias_value":"4HWBXJ2Q2A6D","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_16","alias_value":"4HWBXJ2Q2A6DLL7O","created_at":"2026-07-05T07:09:59Z"},{"alias_kind":"pith_short_8","alias_value":"4HWBXJ2Q","created_at":"2026-07-05T07:09:59Z"}],"graph_snapshots":[{"event_id":"sha256:880a23ffefda293762bbdc1a2608a508794b5eec88f87f6dd7ab59b6ca1c8ff7","target":"graph","created_at":"2026-07-05T07:09:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.03533/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are increasingly used as powerful tools for several high-stakes natural language processing (NLP) applications. Recent prompting works claim to elicit intermediate reasoning steps and key tokens that serve as proxy explanations for LLM predictions. However, there is no certainty whether these explanations are reliable and reflect the LLMs behavior. In this work, we make one of the first attempts at quantifying the uncertainty in explanations of LLMs. To this end, we propose two novel metrics -- $\\textit{Verbalized Uncertainty}$ and $\\textit{Probing Uncertainty}$ --","authors_text":"Chirag Agarwal, Himabindu Lakkaraju, Sree Harsha Tanneru","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-06T21:14:40Z","title":"Quantifying Uncertainty in Natural Language Explanations of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.03533","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cb6e7b7313ffa14ed89bccb4ec2ce06dfb101b680ef962cfa6a33e5db963c31a","target":"record","created_at":"2026-07-05T07:09:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fd183c1db094f111573892b243458cc8b94e8efc72479e20919c7a360732aac8","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-06T21:14:40Z","title_canon_sha256":"49e50d297ac680beba66fc50f755789cf17409e5ace361d3cc14faac64ce33c5"},"schema_version":"1.0","source":{"id":"2311.03533","kind":"arxiv","version":1}},"canonical_sha256":"e1ec1ba750d03c35afee5f0a770ef1bcf3a78b56c246193c30708a0951b4e932","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e1ec1ba750d03c35afee5f0a770ef1bcf3a78b56c246193c30708a0951b4e932","first_computed_at":"2026-07-05T07:09:59.758853Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:09:59.758853Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"t4E1bthd2+4lSYh6mebgV+Xm4LEZRTAB/PVp8vx4anq/65+i8bMffQVoxzENbQhTiXcLMJQhCrKYka9IW4jHDA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:09:59.759499Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.03533","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cb6e7b7313ffa14ed89bccb4ec2ce06dfb101b680ef962cfa6a33e5db963c31a","sha256:880a23ffefda293762bbdc1a2608a508794b5eec88f87f6dd7ab59b6ca1c8ff7"],"state_sha256":"4869c8fc7a97668aa6a034f9e9740ccb22a9e91d4938c8a348d8d5ec2274607c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tTEpBEwIzp+ofDrD9o2JGQLoSSLkPPF6SOCFg38xcHmTHDbN6xLH517sum6OBQ5FVQk02sLRTmxcyJXUp/S/DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T07:18:12.247964Z","bundle_sha256":"d1a2acab7edebf9917fa747e3a44435020c7d33cc9db9f01d2a84c107c822ccd"}}