{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:NX2OQTXMCRXTAEHJX5G7BDFDTK","short_pith_number":"pith:NX2OQTXM","canonical_record":{"source":{"id":"2408.13704","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-25T02:01:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"b69f0b7092c82cf17449c4ed2eaa248ee7c948baee7262e5802522f91d87d2aa","abstract_canon_sha256":"66519a15b6e932778f30f02ac165f93500d1043db0da941344d39a2a73ff87bd"},"schema_version":"1.0"},"canonical_sha256":"6df4e84eec146f3010e9bf4df08ca39aa1729f06b1f2a9489e96ad338dac27b4","source":{"kind":"arxiv","id":"2408.13704","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.13704","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"arxiv_version","alias_value":"2408.13704v2","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.13704","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_12","alias_value":"NX2OQTXMCRXT","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_16","alias_value":"NX2OQTXMCRXTAEHJ","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_8","alias_value":"NX2OQTXM","created_at":"2026-07-05T10:19:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:NX2OQTXMCRXTAEHJX5G7BDFDTK","target":"record","payload":{"canonical_record":{"source":{"id":"2408.13704","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-25T02:01:38Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"b69f0b7092c82cf17449c4ed2eaa248ee7c948baee7262e5802522f91d87d2aa","abstract_canon_sha256":"66519a15b6e932778f30f02ac165f93500d1043db0da941344d39a2a73ff87bd"},"schema_version":"1.0"},"canonical_sha256":"6df4e84eec146f3010e9bf4df08ca39aa1729f06b1f2a9489e96ad338dac27b4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:19:27.256462Z","signature_b64":"eUhCbaGJRdkeQD8PGLrPd0xgjDo8iXkrUVMjPSDwUOWONHlETopw7q3ER3m3zL30YCOKIVVsUmaqIYG/kfFCBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6df4e84eec146f3010e9bf4df08ca39aa1729f06b1f2a9489e96ad338dac27b4","last_reissued_at":"2026-07-05T10:19:27.255942Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:19:27.255942Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.13704","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:19:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lzdWQcpirBfwAKosFZeDT/2TOa6xu+w/Ra0w1TRY0KWIOmknAfgC3b293/8nXpYUwt0voa08ofC4UYcb74F3BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T11:07:09.736958Z"},"content_sha256":"90744f30f1fe67c75be2b6f3e796cd7c0f30941637c2c9203339ad481e75a757","schema_version":"1.0","event_id":"sha256:90744f30f1fe67c75be2b6f3e796cd7c0f30941637c2c9203339ad481e75a757"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:NX2OQTXMCRXTAEHJX5G7BDFDTK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"DHP Benchmark: Are LLMs Good NLG Evaluators?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Jiayi Yuan, Mark Cusick, Param Kulkarni, Xia Hu, Yasser Ibrahim, Yicheng Wang, Yingchi Liu, Yu-Neng Chuang, Zhengping Ji, Zhuoer Wang","submitted_at":"2024-08-25T02:01:38Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly serving as evaluators in Natural Language Generation (NLG) tasks; this is often referred to as ``LLM-as-a-judge'' paradigm. However, the capabilities of LLMs in evaluating NLG quality remain underexplored. Current studies depend on human assessments and simple metrics that fail to capture the discernment of LLMs across diverse NLG tasks. To address this gap, we propose the Discernment of Hierarchical Perturbation (DHP) benchmarking framework, which provides quantitative discernment scores for LLMs. This framework leverages hierarchically perturbed "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.13704","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.13704/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:19:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mMuJh9z56suShAC/K2sCbu+3eBc8G9bw0RZ8PrZXDgmqO/PIbxYOI700KmNlQH3HTaDspNj5zw4atYFV59EJAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T11:07:09.737594Z"},"content_sha256":"16c81927e25a814b98097a68a7aff9e71e86beb5d1c86dafcbe0019b9e13b920","schema_version":"1.0","event_id":"sha256:16c81927e25a814b98097a68a7aff9e71e86beb5d1c86dafcbe0019b9e13b920"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/bundle.json","state_url":"https://pith.science/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T11:07:09Z","links":{"resolver":"https://pith.science/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK","bundle":"https://pith.science/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/bundle.json","state":"https://pith.science/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NX2OQTXMCRXTAEHJX5G7BDFDTK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NX2OQTXMCRXTAEHJX5G7BDFDTK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"66519a15b6e932778f30f02ac165f93500d1043db0da941344d39a2a73ff87bd","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-25T02:01:38Z","title_canon_sha256":"b69f0b7092c82cf17449c4ed2eaa248ee7c948baee7262e5802522f91d87d2aa"},"schema_version":"1.0","source":{"id":"2408.13704","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.13704","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"arxiv_version","alias_value":"2408.13704v2","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.13704","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_12","alias_value":"NX2OQTXMCRXT","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_16","alias_value":"NX2OQTXMCRXTAEHJ","created_at":"2026-07-05T10:19:27Z"},{"alias_kind":"pith_short_8","alias_value":"NX2OQTXM","created_at":"2026-07-05T10:19:27Z"}],"graph_snapshots":[{"event_id":"sha256:16c81927e25a814b98097a68a7aff9e71e86beb5d1c86dafcbe0019b9e13b920","target":"graph","created_at":"2026-07-05T10:19:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.13704/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are increasingly serving as evaluators in Natural Language Generation (NLG) tasks; this is often referred to as ``LLM-as-a-judge'' paradigm. However, the capabilities of LLMs in evaluating NLG quality remain underexplored. Current studies depend on human assessments and simple metrics that fail to capture the discernment of LLMs across diverse NLG tasks. To address this gap, we propose the Discernment of Hierarchical Perturbation (DHP) benchmarking framework, which provides quantitative discernment scores for LLMs. This framework leverages hierarchically perturbed ","authors_text":"Jiayi Yuan, Mark Cusick, Param Kulkarni, Xia Hu, Yasser Ibrahim, Yicheng Wang, Yingchi Liu, Yu-Neng Chuang, Zhengping Ji, Zhuoer Wang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-25T02:01:38Z","title":"DHP Benchmark: Are LLMs Good NLG Evaluators?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.13704","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:90744f30f1fe67c75be2b6f3e796cd7c0f30941637c2c9203339ad481e75a757","target":"record","created_at":"2026-07-05T10:19:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"66519a15b6e932778f30f02ac165f93500d1043db0da941344d39a2a73ff87bd","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-25T02:01:38Z","title_canon_sha256":"b69f0b7092c82cf17449c4ed2eaa248ee7c948baee7262e5802522f91d87d2aa"},"schema_version":"1.0","source":{"id":"2408.13704","kind":"arxiv","version":2}},"canonical_sha256":"6df4e84eec146f3010e9bf4df08ca39aa1729f06b1f2a9489e96ad338dac27b4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6df4e84eec146f3010e9bf4df08ca39aa1729f06b1f2a9489e96ad338dac27b4","first_computed_at":"2026-07-05T10:19:27.255942Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:19:27.255942Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"eUhCbaGJRdkeQD8PGLrPd0xgjDo8iXkrUVMjPSDwUOWONHlETopw7q3ER3m3zL30YCOKIVVsUmaqIYG/kfFCBw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:19:27.256462Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.13704","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:90744f30f1fe67c75be2b6f3e796cd7c0f30941637c2c9203339ad481e75a757","sha256:16c81927e25a814b98097a68a7aff9e71e86beb5d1c86dafcbe0019b9e13b920"],"state_sha256":"deba0fb781b9fc74336e2519c7fac1dba399e96798394b1c75e89139253a42a5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"M5xyPeS+GcOVn4W6V3kPk6LW+o0w+SXluuHXHCo7uFTYl2DfJo6AEwQH3w/TO4t27Pim0JsWMjMcHJ8Nf7OYCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T11:07:09.743788Z","bundle_sha256":"21814327a859df1f366a0810bbcdc43c8f4cee512c11985797e7861c7ce7cc03"}}