{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:P7GY733NEQEKOK32YONBPG2OZB","short_pith_number":"pith:P7GY733N","canonical_record":{"source":{"id":"2403.05266","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-03-08T12:42:36Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"ad4d6e41bf67b746919dd8712b60797c6a603c48013c5a95883ed5b9596cc15b","abstract_canon_sha256":"c3ee2d0cb51a477ec70ba4997f0e08f19bafc5de36fde80d451fa4272b09c5ab"},"schema_version":"1.0"},"canonical_sha256":"7fcd8fef6d2408a72b7ac39a179b4ec84380e90ca04698fa93b82db8f0c8e358","source":{"kind":"arxiv","id":"2403.05266","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.05266","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"arxiv_version","alias_value":"2403.05266v3","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.05266","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_12","alias_value":"P7GY733NEQEK","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_16","alias_value":"P7GY733NEQEKOK32","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_8","alias_value":"P7GY733N","created_at":"2026-07-05T09:30:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:P7GY733NEQEKOK32YONBPG2OZB","target":"record","payload":{"canonical_record":{"source":{"id":"2403.05266","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-03-08T12:42:36Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"ad4d6e41bf67b746919dd8712b60797c6a603c48013c5a95883ed5b9596cc15b","abstract_canon_sha256":"c3ee2d0cb51a477ec70ba4997f0e08f19bafc5de36fde80d451fa4272b09c5ab"},"schema_version":"1.0"},"canonical_sha256":"7fcd8fef6d2408a72b7ac39a179b4ec84380e90ca04698fa93b82db8f0c8e358","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:30:09.173068Z","signature_b64":"+Ghph0RMyZ2mPbnZnCheLu1bf75vgB3cr4gj1ktwjad5FyTS945qSgl+2IVsTGY3LbYq370pv62ys8PmyPyoCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7fcd8fef6d2408a72b7ac39a179b4ec84380e90ca04698fa93b82db8f0c8e358","last_reissued_at":"2026-07-05T09:30:09.172591Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:30:09.172591Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.05266","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:30:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aW/4n2m+2KPkaEX6UpMQf/I0i5zLAhiIuNW+h++aU7SsppTBL5rQPYIQDk5bVkEABCDVE6ifWPTsZdJl1LpDCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T06:04:11.933448Z"},"content_sha256":"479158bbf005d497aca218c88341edf2538106dd6c1aee642e50248dc61afeeb","schema_version":"1.0","event_id":"sha256:479158bbf005d497aca218c88341edf2538106dd6c1aee642e50248dc61afeeb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:P7GY733NEQEKOK32YONBPG2OZB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ERBench: An Entity-Relationship based Automatically Verifiable Hallucination Benchmark for Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Jindong Wang, Jio Oh, Junseok Seo, Ruochen Xu, Soyeon Kim, Steven Euijong Whang, Xing Xie","submitted_at":"2024-03-08T12:42:36Z","abstract_excerpt":"Large language models (LLMs) have achieved unprecedented performances in various applications, yet evaluating them is still challenging. Existing benchmarks are either manually constructed or are automatic, but lack the ability to evaluate the thought process of LLMs with arbitrary complexity. We contend that utilizing existing relational databases based on the entity-relationship (ER) model is a promising approach for constructing benchmarks as they contain structured knowledge that can be used to question LLMs. Unlike knowledge graphs, which are also used to evaluate LLMs, relational databas"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.05266","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.05266/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:30:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CMG6C8UPfTIAEpocenbiMcSJiW4EzmBwN363LKPQ+wKE54fNc+nOCSweFi5qjBDo4awwA2Dhe83XSwW2qcW7Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T06:04:11.934051Z"},"content_sha256":"60d258ba5bf79a872ec828c3549b21b83447c6ffb5cf5dae32cc493275a85e0c","schema_version":"1.0","event_id":"sha256:60d258ba5bf79a872ec828c3549b21b83447c6ffb5cf5dae32cc493275a85e0c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/P7GY733NEQEKOK32YONBPG2OZB/bundle.json","state_url":"https://pith.science/pith/P7GY733NEQEKOK32YONBPG2OZB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/P7GY733NEQEKOK32YONBPG2OZB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-17T06:04:11Z","links":{"resolver":"https://pith.science/pith/P7GY733NEQEKOK32YONBPG2OZB","bundle":"https://pith.science/pith/P7GY733NEQEKOK32YONBPG2OZB/bundle.json","state":"https://pith.science/pith/P7GY733NEQEKOK32YONBPG2OZB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/P7GY733NEQEKOK32YONBPG2OZB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:P7GY733NEQEKOK32YONBPG2OZB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c3ee2d0cb51a477ec70ba4997f0e08f19bafc5de36fde80d451fa4272b09c5ab","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-03-08T12:42:36Z","title_canon_sha256":"ad4d6e41bf67b746919dd8712b60797c6a603c48013c5a95883ed5b9596cc15b"},"schema_version":"1.0","source":{"id":"2403.05266","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.05266","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"arxiv_version","alias_value":"2403.05266v3","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.05266","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_12","alias_value":"P7GY733NEQEK","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_16","alias_value":"P7GY733NEQEKOK32","created_at":"2026-07-05T09:30:09Z"},{"alias_kind":"pith_short_8","alias_value":"P7GY733N","created_at":"2026-07-05T09:30:09Z"}],"graph_snapshots":[{"event_id":"sha256:60d258ba5bf79a872ec828c3549b21b83447c6ffb5cf5dae32cc493275a85e0c","target":"graph","created_at":"2026-07-05T09:30:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.05266/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have achieved unprecedented performances in various applications, yet evaluating them is still challenging. Existing benchmarks are either manually constructed or are automatic, but lack the ability to evaluate the thought process of LLMs with arbitrary complexity. We contend that utilizing existing relational databases based on the entity-relationship (ER) model is a promising approach for constructing benchmarks as they contain structured knowledge that can be used to question LLMs. Unlike knowledge graphs, which are also used to evaluate LLMs, relational databas","authors_text":"Jindong Wang, Jio Oh, Junseok Seo, Ruochen Xu, Soyeon Kim, Steven Euijong Whang, Xing Xie","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-03-08T12:42:36Z","title":"ERBench: An Entity-Relationship based Automatically Verifiable Hallucination Benchmark for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.05266","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:479158bbf005d497aca218c88341edf2538106dd6c1aee642e50248dc61afeeb","target":"record","created_at":"2026-07-05T09:30:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c3ee2d0cb51a477ec70ba4997f0e08f19bafc5de36fde80d451fa4272b09c5ab","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-03-08T12:42:36Z","title_canon_sha256":"ad4d6e41bf67b746919dd8712b60797c6a603c48013c5a95883ed5b9596cc15b"},"schema_version":"1.0","source":{"id":"2403.05266","kind":"arxiv","version":3}},"canonical_sha256":"7fcd8fef6d2408a72b7ac39a179b4ec84380e90ca04698fa93b82db8f0c8e358","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7fcd8fef6d2408a72b7ac39a179b4ec84380e90ca04698fa93b82db8f0c8e358","first_computed_at":"2026-07-05T09:30:09.172591Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:30:09.172591Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+Ghph0RMyZ2mPbnZnCheLu1bf75vgB3cr4gj1ktwjad5FyTS945qSgl+2IVsTGY3LbYq370pv62ys8PmyPyoCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:30:09.173068Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.05266","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:479158bbf005d497aca218c88341edf2538106dd6c1aee642e50248dc61afeeb","sha256:60d258ba5bf79a872ec828c3549b21b83447c6ffb5cf5dae32cc493275a85e0c"],"state_sha256":"aaf19527c6ff78b2689a44421680e86e71a14b9b28d0e634be55ce58d3dc8328"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ixD91xrpVMOK/WPiRV5zLMb0HR0r+WglzK0BD9Cz5pgyqOR3Jt8kvxbsXDKAI65LAb1owj1WOKgwFIUex/xiCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-17T06:04:11.938432Z","bundle_sha256":"bd7f7b3d49a361a8e663d3abd53e7e4eb5be39e05bb34f93465be4a874191d62"}}