{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:LJH3G4D2S6YHI4AZ7K6DY5HS7A","short_pith_number":"pith:LJH3G4D2","canonical_record":{"source":{"id":"2404.16164","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T19:40:01Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"656deedf8fb62ee9109f8c1cf59c3e99f0ef9f955c3a38d1b599071c5f610c5f","abstract_canon_sha256":"be75bc3d3d8c843d77f49a8124c30fccd8ae8eae5a9ea92b10ee56848b657ea6"},"schema_version":"1.0"},"canonical_sha256":"5a4fb3707a97b0747019fabc3c74f2f8173fb709b36859436652ec1b3979241e","source":{"kind":"arxiv","id":"2404.16164","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.16164","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"arxiv_version","alias_value":"2404.16164v1","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.16164","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_12","alias_value":"LJH3G4D2S6YH","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_16","alias_value":"LJH3G4D2S6YHI4AZ","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_8","alias_value":"LJH3G4D2","created_at":"2026-07-05T08:12:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:LJH3G4D2S6YHI4AZ7K6DY5HS7A","target":"record","payload":{"canonical_record":{"source":{"id":"2404.16164","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T19:40:01Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"656deedf8fb62ee9109f8c1cf59c3e99f0ef9f955c3a38d1b599071c5f610c5f","abstract_canon_sha256":"be75bc3d3d8c843d77f49a8124c30fccd8ae8eae5a9ea92b10ee56848b657ea6"},"schema_version":"1.0"},"canonical_sha256":"5a4fb3707a97b0747019fabc3c74f2f8173fb709b36859436652ec1b3979241e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:12:02.120561Z","signature_b64":"eQ/12DKtIjVdNx1uFxVH0iG/jKOiBImAAWYoJWeqsDgcfdaOOkTHBCVoa/r1sw4nHIT3SuKiaHyGFo0BvmpDAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5a4fb3707a97b0747019fabc3c74f2f8173fb709b36859436652ec1b3979241e","last_reissued_at":"2026-07-05T08:12:02.120137Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:12:02.120137Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.16164","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:12:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Q1svpuovk1bE/iUb+taoaEK8dbXkA3IoMuwlk3pjfnMiUvFf+/Cz6c1XtcBJyn+Nq/YNmP9ZZBxiPv7HvNkzAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T04:53:18.150097Z"},"content_sha256":"033287d0a8e929b802dc4fbc5cb2b0ccf70eb6caf3bf81346fcc8e3d878a0dfa","schema_version":"1.0","event_id":"sha256:033287d0a8e929b802dc4fbc5cb2b0ccf70eb6caf3bf81346fcc8e3d878a0dfa"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:LJH3G4D2S6YHI4AZ7K6DY5HS7A","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Bonan Min, Chung-Wei Hang, Jiang Guo, Jiaqing Yuan, Jiarong Jiang, Lin Pan, Patrick Ng, Zhiguo Wang","submitted_at":"2024-04-24T19:40:01Z","abstract_excerpt":"Large language models (LLMs) have shown remarkable performance on a variety of NLP tasks, and are being rapidly adopted in a wide range of use cases. It is therefore of vital importance to holistically evaluate the factuality of their generated outputs, as hallucinations remain a challenging issue.\n  In this work, we focus on assessing LLMs' ability to recall factual knowledge learned from pretraining, and the factors that affect this ability. To that end, we construct FACT-BENCH, a representative benchmark covering 20 domains, 134 property types, 3 answer types, and different knowledge popula"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.16164","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.16164/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:12:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9P4+dmeKnuP9WNaWOXU8qJq7c0l81drcBa7CvlgIRBe3BrVMNHYH/1rq7F+jtC9jhgjgP9tBVm5mB+7UWjwtBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T04:53:18.150628Z"},"content_sha256":"fa0b1291d2c9f80b8b31785c81960429a6f860401b2db50a46191f9648327fdf","schema_version":"1.0","event_id":"sha256:fa0b1291d2c9f80b8b31785c81960429a6f860401b2db50a46191f9648327fdf"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/bundle.json","state_url":"https://pith.science/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T04:53:18Z","links":{"resolver":"https://pith.science/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A","bundle":"https://pith.science/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/bundle.json","state":"https://pith.science/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LJH3G4D2S6YHI4AZ7K6DY5HS7A/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LJH3G4D2S6YHI4AZ7K6DY5HS7A","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"be75bc3d3d8c843d77f49a8124c30fccd8ae8eae5a9ea92b10ee56848b657ea6","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T19:40:01Z","title_canon_sha256":"656deedf8fb62ee9109f8c1cf59c3e99f0ef9f955c3a38d1b599071c5f610c5f"},"schema_version":"1.0","source":{"id":"2404.16164","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.16164","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"arxiv_version","alias_value":"2404.16164v1","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.16164","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_12","alias_value":"LJH3G4D2S6YH","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_16","alias_value":"LJH3G4D2S6YHI4AZ","created_at":"2026-07-05T08:12:02Z"},{"alias_kind":"pith_short_8","alias_value":"LJH3G4D2","created_at":"2026-07-05T08:12:02Z"}],"graph_snapshots":[{"event_id":"sha256:fa0b1291d2c9f80b8b31785c81960429a6f860401b2db50a46191f9648327fdf","target":"graph","created_at":"2026-07-05T08:12:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.16164/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have shown remarkable performance on a variety of NLP tasks, and are being rapidly adopted in a wide range of use cases. It is therefore of vital importance to holistically evaluate the factuality of their generated outputs, as hallucinations remain a challenging issue.\n  In this work, we focus on assessing LLMs' ability to recall factual knowledge learned from pretraining, and the factors that affect this ability. To that end, we construct FACT-BENCH, a representative benchmark covering 20 domains, 134 property types, 3 answer types, and different knowledge popula","authors_text":"Bonan Min, Chung-Wei Hang, Jiang Guo, Jiaqing Yuan, Jiarong Jiang, Lin Pan, Patrick Ng, Zhiguo Wang","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T19:40:01Z","title":"Towards a Holistic Evaluation of LLMs on Factual Knowledge Recall"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.16164","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:033287d0a8e929b802dc4fbc5cb2b0ccf70eb6caf3bf81346fcc8e3d878a0dfa","target":"record","created_at":"2026-07-05T08:12:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"be75bc3d3d8c843d77f49a8124c30fccd8ae8eae5a9ea92b10ee56848b657ea6","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T19:40:01Z","title_canon_sha256":"656deedf8fb62ee9109f8c1cf59c3e99f0ef9f955c3a38d1b599071c5f610c5f"},"schema_version":"1.0","source":{"id":"2404.16164","kind":"arxiv","version":1}},"canonical_sha256":"5a4fb3707a97b0747019fabc3c74f2f8173fb709b36859436652ec1b3979241e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5a4fb3707a97b0747019fabc3c74f2f8173fb709b36859436652ec1b3979241e","first_computed_at":"2026-07-05T08:12:02.120137Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:12:02.120137Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"eQ/12DKtIjVdNx1uFxVH0iG/jKOiBImAAWYoJWeqsDgcfdaOOkTHBCVoa/r1sw4nHIT3SuKiaHyGFo0BvmpDAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:12:02.120561Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.16164","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:033287d0a8e929b802dc4fbc5cb2b0ccf70eb6caf3bf81346fcc8e3d878a0dfa","sha256:fa0b1291d2c9f80b8b31785c81960429a6f860401b2db50a46191f9648327fdf"],"state_sha256":"4d530e1418c760f79f419f4c1278abb48d0f12334c82e2fb9ab018a2384dc112"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sIjkjAjU5Py/lbffu8rNU8u8lxuLASMziaQEpGDKT85pmGh3/8eaH09AJfFEgGur3K4aFNASzhPK50sn3sI3DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T04:53:18.155822Z","bundle_sha256":"8f6738d99c8911142d5a95c015e57bec8f437f6edbee7a5693801411641db4ae"}}