{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:JCFDNISWHQRBOHNI3XPUDXLDXD","short_pith_number":"pith:JCFDNISW","canonical_record":{"source":{"id":"2405.19740","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T06:38:32Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"92ecbd7fb16027edf309db4f19133e80d8e001aba9f9b091ec15d874b676d35e","abstract_canon_sha256":"5475cc402f8af777e2794fc432cb0ff30219d7d7ffad55b5e574d12aceade6fe"},"schema_version":"1.0"},"canonical_sha256":"488a36a2563c22171da8dddf41dd63b8dc48018476c89e0cc30ad4f038eb3843","source":{"kind":"arxiv","id":"2405.19740","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19740","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19740v2","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19740","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_12","alias_value":"JCFDNISWHQRB","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_16","alias_value":"JCFDNISWHQRBOHNI","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_8","alias_value":"JCFDNISW","created_at":"2026-07-05T09:22:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:JCFDNISWHQRBOHNI3XPUDXLDXD","target":"record","payload":{"canonical_record":{"source":{"id":"2405.19740","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T06:38:32Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"92ecbd7fb16027edf309db4f19133e80d8e001aba9f9b091ec15d874b676d35e","abstract_canon_sha256":"5475cc402f8af777e2794fc432cb0ff30219d7d7ffad55b5e574d12aceade6fe"},"schema_version":"1.0"},"canonical_sha256":"488a36a2563c22171da8dddf41dd63b8dc48018476c89e0cc30ad4f038eb3843","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:22:18.594702Z","signature_b64":"L7UttfI+yfhxFvWi0wyE5IajQ5H6CMYg1CbGbygHWmE7ec3oa0DRMxdNO+G3UtyWJcwu6rCOBUNersaAmWm9BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"488a36a2563c22171da8dddf41dd63b8dc48018476c89e0cc30ad4f038eb3843","last_reissued_at":"2026-07-05T09:22:18.594215Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:22:18.594215Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.19740","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T6X1Z7JH+RNgER/Rs4aQiYX9SUb8yGp6IPZ0KDyDK/yTBRmU/uFf+K07siMcFVtF1rRwxfi+TJ+dm5tXl13/CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T01:01:35.391653Z"},"content_sha256":"33719c3c9452f7fb8979e708d4bcb4080eb26075910476e67d6ad0309118bbdc","schema_version":"1.0","event_id":"sha256:33719c3c9452f7fb8979e708d4bcb4080eb26075910476e67d6ad0309118bbdc"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:JCFDNISWHQRBOHNI3XPUDXLDXD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PertEval: Unveiling Real Knowledge Capacity of LLMs with Knowledge-Invariant Perturbations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CY"],"primary_cat":"cs.CL","authors_text":"Jiatong Li, Kunzhe Huang, Mengxiao Zhu, Qi Liu, Renjun Hu, Wei Lin, Xing Shi, Yan Zhuang","submitted_at":"2024-05-30T06:38:32Z","abstract_excerpt":"Expert-designed close-ended benchmarks are indispensable in assessing the knowledge capacity of large language models (LLMs). Despite their widespread use, concerns have mounted regarding their reliability due to limited test scenarios and an unavoidable risk of data contamination. To rectify this, we present PertEval, a toolkit devised for in-depth probing of LLMs' knowledge capacity through \\textbf{knowledge-invariant perturbations}. These perturbations employ human-like restatement techniques to generate on-the-fly test samples from static benchmarks, meticulously retaining knowledge-critic"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19740","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.19740/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3WxMm/lA9gdukbrEBYdAHsS/KmRF1In5DDdM4WwjBhDrgu1spegPWcBqq+L6aKE02IbTVfbCA3526zV8xkjvCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T01:01:35.392564Z"},"content_sha256":"928f8c49e38aa7bb543774f05c46aeac69a2f06909d60e9313051709a686b3d7","schema_version":"1.0","event_id":"sha256:928f8c49e38aa7bb543774f05c46aeac69a2f06909d60e9313051709a686b3d7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/bundle.json","state_url":"https://pith.science/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T01:01:35Z","links":{"resolver":"https://pith.science/pith/JCFDNISWHQRBOHNI3XPUDXLDXD","bundle":"https://pith.science/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/bundle.json","state":"https://pith.science/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JCFDNISWHQRBOHNI3XPUDXLDXD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:JCFDNISWHQRBOHNI3XPUDXLDXD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5475cc402f8af777e2794fc432cb0ff30219d7d7ffad55b5e574d12aceade6fe","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T06:38:32Z","title_canon_sha256":"92ecbd7fb16027edf309db4f19133e80d8e001aba9f9b091ec15d874b676d35e"},"schema_version":"1.0","source":{"id":"2405.19740","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.19740","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"arxiv_version","alias_value":"2405.19740v2","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.19740","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_12","alias_value":"JCFDNISWHQRB","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_16","alias_value":"JCFDNISWHQRBOHNI","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_8","alias_value":"JCFDNISW","created_at":"2026-07-05T09:22:18Z"}],"graph_snapshots":[{"event_id":"sha256:928f8c49e38aa7bb543774f05c46aeac69a2f06909d60e9313051709a686b3d7","target":"graph","created_at":"2026-07-05T09:22:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.19740/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Expert-designed close-ended benchmarks are indispensable in assessing the knowledge capacity of large language models (LLMs). Despite their widespread use, concerns have mounted regarding their reliability due to limited test scenarios and an unavoidable risk of data contamination. To rectify this, we present PertEval, a toolkit devised for in-depth probing of LLMs' knowledge capacity through \\textbf{knowledge-invariant perturbations}. These perturbations employ human-like restatement techniques to generate on-the-fly test samples from static benchmarks, meticulously retaining knowledge-critic","authors_text":"Jiatong Li, Kunzhe Huang, Mengxiao Zhu, Qi Liu, Renjun Hu, Wei Lin, Xing Shi, Yan Zhuang","cross_cats":["cs.AI","cs.CY"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T06:38:32Z","title":"PertEval: Unveiling Real Knowledge Capacity of LLMs with Knowledge-Invariant Perturbations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.19740","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:33719c3c9452f7fb8979e708d4bcb4080eb26075910476e67d6ad0309118bbdc","target":"record","created_at":"2026-07-05T09:22:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5475cc402f8af777e2794fc432cb0ff30219d7d7ffad55b5e574d12aceade6fe","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-30T06:38:32Z","title_canon_sha256":"92ecbd7fb16027edf309db4f19133e80d8e001aba9f9b091ec15d874b676d35e"},"schema_version":"1.0","source":{"id":"2405.19740","kind":"arxiv","version":2}},"canonical_sha256":"488a36a2563c22171da8dddf41dd63b8dc48018476c89e0cc30ad4f038eb3843","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"488a36a2563c22171da8dddf41dd63b8dc48018476c89e0cc30ad4f038eb3843","first_computed_at":"2026-07-05T09:22:18.594215Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:22:18.594215Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"L7UttfI+yfhxFvWi0wyE5IajQ5H6CMYg1CbGbygHWmE7ec3oa0DRMxdNO+G3UtyWJcwu6rCOBUNersaAmWm9BA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:22:18.594702Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.19740","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:33719c3c9452f7fb8979e708d4bcb4080eb26075910476e67d6ad0309118bbdc","sha256:928f8c49e38aa7bb543774f05c46aeac69a2f06909d60e9313051709a686b3d7"],"state_sha256":"bbef6d799eda87d4a6a9aef678ee106ec927204b9469e5efc7df1d229b054f94"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H6gqYv4ZFZ7YGYoP6f7+UloDieYkBNBw7Y8EZryTn3SNEC1R5BvxzBT4b6KxxsKIo56zSyLWpc+wHKFw5GKWDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T01:01:35.402531Z","bundle_sha256":"d13a449b1b56927656b26378ba8fe7fb743b01007101e8cd06eda3691338b067"}}