{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:3ALB7XMRWXIPXE5NE7YOTENQ3S","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6ec54b336a01a0b7694b39f6c09402f5cf0b092de5e3c7154b60b16c9fe2bb47","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-30T17:04:35Z","title_canon_sha256":"ad921c7c2e23938c7cd59e57acdad2373fb9848fa649e28accfee761e3370b1a"},"schema_version":"1.0","source":{"id":"2310.19740","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.19740","created_at":"2026-07-05T10:02:21Z"},{"alias_kind":"arxiv_version","alias_value":"2310.19740v2","created_at":"2026-07-05T10:02:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.19740","created_at":"2026-07-05T10:02:21Z"},{"alias_kind":"pith_short_12","alias_value":"3ALB7XMRWXIP","created_at":"2026-07-05T10:02:21Z"},{"alias_kind":"pith_short_16","alias_value":"3ALB7XMRWXIPXE5N","created_at":"2026-07-05T10:02:21Z"},{"alias_kind":"pith_short_8","alias_value":"3ALB7XMR","created_at":"2026-07-05T10:02:21Z"}],"graph_snapshots":[{"event_id":"sha256:a8a039a1235527bf834a934275e4d4234ffc9eb3f4748d78c365ce1649a735a2","target":"graph","created_at":"2026-07-05T10:02:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.19740/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Previous work adopts large language models (LLMs) as evaluators to evaluate natural language process (NLP) tasks. However, certain shortcomings, e.g., fairness, scope, and accuracy, persist for current LLM evaluators. To analyze whether LLMs can serve as reliable alternatives to humans, we examine the fine-grained alignment between LLM evaluators and human annotators, particularly in understanding the target evaluation tasks and conducting evaluations that meet diverse criteria. This paper explores both conventional tasks (e.g., story generation) and alignment tasks (e.g., math reasoning), eac","authors_text":"Leyang Cui, Lingpeng Kong, Qintong Li, Wei Bi","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-30T17:04:35Z","title":"Exploring the Reliability of Large Language Models as Customized Evaluators for Diverse NLP Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.19740","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:125e4534a94e0162af2b1592c8701691fa3dca42a29f19bd6ef1dea406c083e9","target":"record","created_at":"2026-07-05T10:02:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6ec54b336a01a0b7694b39f6c09402f5cf0b092de5e3c7154b60b16c9fe2bb47","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-10-30T17:04:35Z","title_canon_sha256":"ad921c7c2e23938c7cd59e57acdad2373fb9848fa649e28accfee761e3370b1a"},"schema_version":"1.0","source":{"id":"2310.19740","kind":"arxiv","version":2}},"canonical_sha256":"d8161fdd91b5d0fb93ad27f0e991b0dc9d30c5c3a88c83d8e70abd5924b50732","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d8161fdd91b5d0fb93ad27f0e991b0dc9d30c5c3a88c83d8e70abd5924b50732","first_computed_at":"2026-07-05T10:02:21.708159Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:02:21.708159Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"hZedTSim8K3KYNs5nSUI6xtUE33vRYPbnOaX737kiGiMKXFcLa8UJtOROGxanYSdOMcHSBNBbPLNQqfzRzsCBA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:02:21.708719Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.19740","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:125e4534a94e0162af2b1592c8701691fa3dca42a29f19bd6ef1dea406c083e9","sha256:a8a039a1235527bf834a934275e4d4234ffc9eb3f4748d78c365ce1649a735a2"],"state_sha256":"d5d00b08825f9d62bcdc3f0e6a9185b4834a5ad67c549fb20486656a113a58c6"}