{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WZQEJHWELMDNGRAXSCRWX2QETS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0db082aab681b2d8d2c8153a77abf582d90f0553cb8b0b26a2933e4103b6ca5e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T15:20:18Z","title_canon_sha256":"03d99ae4380450c56c529c4981ed53326fd400cbe6e7ddbdb38a2edaa3e9231f"},"schema_version":"1.0","source":{"id":"2407.13578","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.13578","created_at":"2026-07-05T09:49:44Z"},{"alias_kind":"arxiv_version","alias_value":"2407.13578v2","created_at":"2026-07-05T09:49:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.13578","created_at":"2026-07-05T09:49:44Z"},{"alias_kind":"pith_short_12","alias_value":"WZQEJHWELMDN","created_at":"2026-07-05T09:49:44Z"},{"alias_kind":"pith_short_16","alias_value":"WZQEJHWELMDNGRAX","created_at":"2026-07-05T09:49:44Z"},{"alias_kind":"pith_short_8","alias_value":"WZQEJHWE","created_at":"2026-07-05T09:49:44Z"}],"graph_snapshots":[{"event_id":"sha256:093fcb15e7f8db2645837d6df7da2004e57ea1c851d5e707b5d4ebdcf0c0edb3","target":"graph","created_at":"2026-07-05T09:49:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.13578/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are increasingly explored as knowledge bases (KBs), yet current evaluation methods focus too narrowly on knowledge retention, overlooking other crucial criteria for reliable performance. In this work, we rethink the requirements for evaluating reliable LLM-as-KB usage and highlight two essential factors: factuality, ensuring accurate responses to seen and unseen knowledge, and consistency, maintaining stable answers to questions about the same knowledge. We introduce UnseenQA, a dataset designed to assess LLM performance on unseen knowledge, and propose new criteri","authors_text":"Danna Zheng, Jeff Z. Pan, Mirella Lapata","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T15:20:18Z","title":"How Reliable are LLMs as Knowledge Bases? Re-thinking Facutality and Consistency"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.13578","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:63e286f6b3faf7912247876cfacfbb7510689aa606e1e28fdfea473789f390b0","target":"record","created_at":"2026-07-05T09:49:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0db082aab681b2d8d2c8153a77abf582d90f0553cb8b0b26a2933e4103b6ca5e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T15:20:18Z","title_canon_sha256":"03d99ae4380450c56c529c4981ed53326fd400cbe6e7ddbdb38a2edaa3e9231f"},"schema_version":"1.0","source":{"id":"2407.13578","kind":"arxiv","version":2}},"canonical_sha256":"b660449ec45b06d3441790a36bea049c9731132122bc9c75c56523d667c5db8f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b660449ec45b06d3441790a36bea049c9731132122bc9c75c56523d667c5db8f","first_computed_at":"2026-07-05T09:49:44.011261Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:49:44.011261Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WyhNcSemNzuidkVWWEawP/JSThNw1gSSsZNQskerdttzNnlPp9VHltEDB0EHN/35PVgsAPjgpWiNx0oB3XIPCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:49:44.011767Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.13578","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:63e286f6b3faf7912247876cfacfbb7510689aa606e1e28fdfea473789f390b0","sha256:093fcb15e7f8db2645837d6df7da2004e57ea1c851d5e707b5d4ebdcf0c0edb3"],"state_sha256":"3d24e710f6123aec93b20ba09794f476b63e66cb071d0c7b06d1c873c4564a51"}