{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:Y2MQTELLL2VHRMOSQJSZKV3CJO","short_pith_number":"pith:Y2MQTELL","canonical_record":{"source":{"id":"2310.09820","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-15T12:40:30Z","cross_cats_sorted":[],"title_canon_sha256":"a0845ead12d5da478ec9224642215cc995566b3d2671fcd6d0564ecb1e4c0b10","abstract_canon_sha256":"ed480dc21acf08bb7ab56522aa3d890be609aa234a736392695afc7aa017cd09"},"schema_version":"1.0"},"canonical_sha256":"c69909916b5eaa78b1d282659557624b8126b4947b59af0cca7452ac7eb415e7","source":{"kind":"arxiv","id":"2310.09820","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.09820","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"arxiv_version","alias_value":"2310.09820v1","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.09820","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_12","alias_value":"Y2MQTELLL2VH","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_16","alias_value":"Y2MQTELLL2VHRMOS","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_8","alias_value":"Y2MQTELL","created_at":"2026-07-05T07:01:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:Y2MQTELLL2VHRMOSQJSZKV3CJO","target":"record","payload":{"canonical_record":{"source":{"id":"2310.09820","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-15T12:40:30Z","cross_cats_sorted":[],"title_canon_sha256":"a0845ead12d5da478ec9224642215cc995566b3d2671fcd6d0564ecb1e4c0b10","abstract_canon_sha256":"ed480dc21acf08bb7ab56522aa3d890be609aa234a736392695afc7aa017cd09"},"schema_version":"1.0"},"canonical_sha256":"c69909916b5eaa78b1d282659557624b8126b4947b59af0cca7452ac7eb415e7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:11.610935Z","signature_b64":"RUYyhaR3sQsK5Jaor87hL6Vpv/FgI7kKFgNaqcD4f5X6xU+ZKIswIPwp6e5F5J6Ow6XoH5+YbXOxyM1/H6P5DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c69909916b5eaa78b1d282659557624b8126b4947b59af0cca7452ac7eb415e7","last_reissued_at":"2026-07-05T07:01:11.610548Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:11.610548Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.09820","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Iyr2p5k1xkmQft8wgIVClmIqEinqgLwt227SaQpkAVje/HRTyChzOsTHmjHGfPETwAHn+Idi13cE0AGTkJSxAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T01:43:20.780770Z"},"content_sha256":"bdfdd3957705fd748449fb9789ab1241280e729a2980560bcabe8637e4092d0e","schema_version":"1.0","event_id":"sha256:bdfdd3957705fd748449fb9789ab1241280e729a2980560bcabe8637e4092d0e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:Y2MQTELLL2VHRMOSQJSZKV3CJO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Assessing the Reliability of Large Language Model Knowledge","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexandra Birch, Barry Haddow, Wei Peng, Weixuan Wang","submitted_at":"2023-10-15T12:40:30Z","abstract_excerpt":"Large language models (LLMs) have been treated as knowledge bases due to their strong performance in knowledge probing tasks. LLMs are typically evaluated using accuracy, yet this metric does not capture the vulnerability of LLMs to hallucination-inducing factors like prompt and context variability. How do we evaluate the capabilities of LLMs to consistently produce factually correct answers? In this paper, we propose MOdel kNowledge relIabiliTy scORe (MONITOR), a novel metric designed to directly measure LLMs' factual reliability. MONITOR computes the distance between the probability distribu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.09820","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.09820/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I4v0/ASD7URKZYvJQFXg1rWx3igQ3n+XgzRsbMb0XMSnC4v/MdeRzWkGFW6yqbEB6mjRjqYN5AiAbrV/QUTxCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T01:43:20.781145Z"},"content_sha256":"09af784b96931590d75d484934059b91e9d6708563fe807cf1acf00720d8e6d7","schema_version":"1.0","event_id":"sha256:09af784b96931590d75d484934059b91e9d6708563fe807cf1acf00720d8e6d7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/bundle.json","state_url":"https://pith.science/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T01:43:20Z","links":{"resolver":"https://pith.science/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO","bundle":"https://pith.science/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/bundle.json","state":"https://pith.science/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Y2MQTELLL2VHRMOSQJSZKV3CJO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:Y2MQTELLL2VHRMOSQJSZKV3CJO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ed480dc21acf08bb7ab56522aa3d890be609aa234a736392695afc7aa017cd09","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-15T12:40:30Z","title_canon_sha256":"a0845ead12d5da478ec9224642215cc995566b3d2671fcd6d0564ecb1e4c0b10"},"schema_version":"1.0","source":{"id":"2310.09820","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.09820","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"arxiv_version","alias_value":"2310.09820v1","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.09820","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_12","alias_value":"Y2MQTELLL2VH","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_16","alias_value":"Y2MQTELLL2VHRMOS","created_at":"2026-07-05T07:01:11Z"},{"alias_kind":"pith_short_8","alias_value":"Y2MQTELL","created_at":"2026-07-05T07:01:11Z"}],"graph_snapshots":[{"event_id":"sha256:09af784b96931590d75d484934059b91e9d6708563fe807cf1acf00720d8e6d7","target":"graph","created_at":"2026-07-05T07:01:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.09820/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have been treated as knowledge bases due to their strong performance in knowledge probing tasks. LLMs are typically evaluated using accuracy, yet this metric does not capture the vulnerability of LLMs to hallucination-inducing factors like prompt and context variability. How do we evaluate the capabilities of LLMs to consistently produce factually correct answers? In this paper, we propose MOdel kNowledge relIabiliTy scORe (MONITOR), a novel metric designed to directly measure LLMs' factual reliability. MONITOR computes the distance between the probability distribu","authors_text":"Alexandra Birch, Barry Haddow, Wei Peng, Weixuan Wang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-15T12:40:30Z","title":"Assessing the Reliability of Large Language Model Knowledge"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.09820","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bdfdd3957705fd748449fb9789ab1241280e729a2980560bcabe8637e4092d0e","target":"record","created_at":"2026-07-05T07:01:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ed480dc21acf08bb7ab56522aa3d890be609aa234a736392695afc7aa017cd09","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-15T12:40:30Z","title_canon_sha256":"a0845ead12d5da478ec9224642215cc995566b3d2671fcd6d0564ecb1e4c0b10"},"schema_version":"1.0","source":{"id":"2310.09820","kind":"arxiv","version":1}},"canonical_sha256":"c69909916b5eaa78b1d282659557624b8126b4947b59af0cca7452ac7eb415e7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c69909916b5eaa78b1d282659557624b8126b4947b59af0cca7452ac7eb415e7","first_computed_at":"2026-07-05T07:01:11.610548Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:01:11.610548Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"RUYyhaR3sQsK5Jaor87hL6Vpv/FgI7kKFgNaqcD4f5X6xU+ZKIswIPwp6e5F5J6Ow6XoH5+YbXOxyM1/H6P5DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T07:01:11.610935Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.09820","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bdfdd3957705fd748449fb9789ab1241280e729a2980560bcabe8637e4092d0e","sha256:09af784b96931590d75d484934059b91e9d6708563fe807cf1acf00720d8e6d7"],"state_sha256":"c33cc77fab1780be6783856485d70c3e1cfa4ecd1bf14adf788f92d55494aebf"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"78hXJycJ4V0ENsOL9JOMQ9lZLB88SH9bL/N/W/IsVJysZznY1buCrzfuqGHpMiKbhH1WZOVrQVp3s/dIJ92fAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T01:43:20.783528Z","bundle_sha256":"4464ea17f3a84ff61d29157f6ba81026188b83ce2a1e27e82b15733775420e5b"}}