{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LLEPUGMDRKEDXTJCU26FRIJ5FB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"74e1445e7027fb708763373db8994aeb54aea14bc27d78331f699f505ec1aa2b","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CV","submitted_at":"2024-05-05T12:51:38Z","title_canon_sha256":"c9fada98ce4f11e654800992682bb156fb23032704dc30a95964ba29cd0b1046"},"schema_version":"1.0","source":{"id":"2405.02917","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.02917","created_at":"2026-07-05T08:15:55Z"},{"alias_kind":"arxiv_version","alias_value":"2405.02917v1","created_at":"2026-07-05T08:15:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.02917","created_at":"2026-07-05T08:15:55Z"},{"alias_kind":"pith_short_12","alias_value":"LLEPUGMDRKED","created_at":"2026-07-05T08:15:55Z"},{"alias_kind":"pith_short_16","alias_value":"LLEPUGMDRKEDXTJC","created_at":"2026-07-05T08:15:55Z"},{"alias_kind":"pith_short_8","alias_value":"LLEPUGMD","created_at":"2026-07-05T08:15:55Z"}],"graph_snapshots":[{"event_id":"sha256:ab06b0ce4cf298818010192f8e3dcaac9468fd59983861b975d5a9f043388fb8","target":"graph","created_at":"2026-07-05T08:15:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.02917/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language and Vision-Language Models (LLMs/VLMs) have revolutionized the field of AI by their ability to generate human-like text and understand images, but ensuring their reliability is crucial. This paper aims to evaluate the ability of LLMs (GPT4, GPT-3.5, LLaMA2, and PaLM 2) and VLMs (GPT4V and Gemini Pro Vision) to estimate their verbalized uncertainty via prompting. We propose the new Japanese Uncertain Scenes (JUS) dataset, aimed at testing VLM capabilities via difficult queries and object counting, and the Net Calibration Error (NCE) to measure direction of miscalibration. Results show ","authors_text":"Matias Valdenegro-Toro, Tobias Groot","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CV","submitted_at":"2024-05-05T12:51:38Z","title":"Overconfidence is Key: Verbalized Uncertainty Evaluation in Large Language and Vision-Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.02917","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:76f6652621e9b98267f804ee57b64a7a40d70199e2f6cd7ba87f2468dea2f6be","target":"record","created_at":"2026-07-05T08:15:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"74e1445e7027fb708763373db8994aeb54aea14bc27d78331f699f505ec1aa2b","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CV","submitted_at":"2024-05-05T12:51:38Z","title_canon_sha256":"c9fada98ce4f11e654800992682bb156fb23032704dc30a95964ba29cd0b1046"},"schema_version":"1.0","source":{"id":"2405.02917","kind":"arxiv","version":1}},"canonical_sha256":"5ac8fa19838a883bcd22a6bc58a13d285b3df1ec414ac8a497a6961b8a77258f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5ac8fa19838a883bcd22a6bc58a13d285b3df1ec414ac8a497a6961b8a77258f","first_computed_at":"2026-07-05T08:15:55.849845Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:15:55.849845Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"v1TfiREDtxbV2fqVydPvfq1oW+P+ziT1IshDpTR/XII9shtXxV38p+UrP4Ryob4McZ/59CQJsiAbwm2hG7d1Dw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:15:55.850552Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.02917","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:76f6652621e9b98267f804ee57b64a7a40d70199e2f6cd7ba87f2468dea2f6be","sha256:ab06b0ce4cf298818010192f8e3dcaac9468fd59983861b975d5a9f043388fb8"],"state_sha256":"14845c18b8f454fe72ba24c3d16333ecb4d52bfbcc7eae06b0941108f7151b0b"}