{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:ZQGCJGLGKLALMKB7F2RM2RZHXN","short_pith_number":"pith:ZQGCJGLG","canonical_record":{"source":{"id":"2310.01917","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-03T09:46:02Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"226d9c4334352d6ecc3b05836588b9648bee9b76c2a5079c188b50ffdf53c6fd","abstract_canon_sha256":"00669744024835a16bc96acc60164de2a8d0c8b8634043690f19667c132010ae"},"schema_version":"1.0"},"canonical_sha256":"cc0c24996652c0b6283f2ea2cd4727bb67cf46d7ddd5a17328c7e9e3f78a0aac","source":{"kind":"arxiv","id":"2310.01917","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.01917","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"arxiv_version","alias_value":"2310.01917v2","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.01917","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_12","alias_value":"ZQGCJGLGKLAL","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_16","alias_value":"ZQGCJGLGKLALMKB7","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_8","alias_value":"ZQGCJGLG","created_at":"2026-07-05T06:59:54Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:ZQGCJGLGKLALMKB7F2RM2RZHXN","target":"record","payload":{"canonical_record":{"source":{"id":"2310.01917","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-03T09:46:02Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"226d9c4334352d6ecc3b05836588b9648bee9b76c2a5079c188b50ffdf53c6fd","abstract_canon_sha256":"00669744024835a16bc96acc60164de2a8d0c8b8634043690f19667c132010ae"},"schema_version":"1.0"},"canonical_sha256":"cc0c24996652c0b6283f2ea2cd4727bb67cf46d7ddd5a17328c7e9e3f78a0aac","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:59:54.354442Z","signature_b64":"BJbitDtzTbpte4iWbV3JG9S6UJKLyCNQ9q6SHKE/iiqBFdhzI88AdO8egg1TEN2gve/eF6bU7vGFqkqQeB6MDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cc0c24996652c0b6283f2ea2cd4727bb67cf46d7ddd5a17328c7e9e3f78a0aac","last_reissued_at":"2026-07-05T06:59:54.353975Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:59:54.353975Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.01917","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:59:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DF2zQgF2N4ATOgXk5IgwFnTuAR6bMPhxa3kATK6NaEe2YhTGx103xiHYQfKngnzsMiS0pb2av0kN82VahwUdBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T05:04:04.587099Z"},"content_sha256":"b864e9066c7a4ba957881a8d9c2c8fc106a2ae1e0844781a2154fb401a7bc4b5","schema_version":"1.0","event_id":"sha256:b864e9066c7a4ba957881a8d9c2c8fc106a2ae1e0844781a2154fb401a7bc4b5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:ZQGCJGLGKLALMKB7F2RM2RZHXN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Hierarchical Evaluation Framework: Best Practices for Human Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.CL","authors_text":"Iva Bojic, Jessica Chen, Josip Car, Qi Chwen Ong, Shafiq Joty, Si Yuan Chang","submitted_at":"2023-10-03T09:46:02Z","abstract_excerpt":"Human evaluation plays a crucial role in Natural Language Processing (NLP) as it assesses the quality and relevance of developed systems, thereby facilitating their enhancement. However, the absence of widely accepted human evaluation metrics in NLP hampers fair comparisons among different systems and the establishment of universal assessment standards. Through an extensive analysis of existing literature on human evaluation metrics, we identified several gaps in NLP evaluation methodologies. These gaps served as motivation for developing our own hierarchical evaluation framework. The proposed"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.01917","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.01917/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:59:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BrfTHey0MBT6T5So8qdW8M9St8wIsGHXT2BWniCMcYLOFBgUEEXfmhtjMVtDqOYwY3ZtNi8Dued8R9dT52MSDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T05:04:04.587496Z"},"content_sha256":"18f1ea8c7d8dd7d03fb1708e4d44ca7b62626c3b27f234e4b86d11ffbbbc16ce","schema_version":"1.0","event_id":"sha256:18f1ea8c7d8dd7d03fb1708e4d44ca7b62626c3b27f234e4b86d11ffbbbc16ce"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/bundle.json","state_url":"https://pith.science/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T05:04:04Z","links":{"resolver":"https://pith.science/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN","bundle":"https://pith.science/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/bundle.json","state":"https://pith.science/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZQGCJGLGKLALMKB7F2RM2RZHXN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:ZQGCJGLGKLALMKB7F2RM2RZHXN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"00669744024835a16bc96acc60164de2a8d0c8b8634043690f19667c132010ae","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-03T09:46:02Z","title_canon_sha256":"226d9c4334352d6ecc3b05836588b9648bee9b76c2a5079c188b50ffdf53c6fd"},"schema_version":"1.0","source":{"id":"2310.01917","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.01917","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"arxiv_version","alias_value":"2310.01917v2","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.01917","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_12","alias_value":"ZQGCJGLGKLAL","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_16","alias_value":"ZQGCJGLGKLALMKB7","created_at":"2026-07-05T06:59:54Z"},{"alias_kind":"pith_short_8","alias_value":"ZQGCJGLG","created_at":"2026-07-05T06:59:54Z"}],"graph_snapshots":[{"event_id":"sha256:18f1ea8c7d8dd7d03fb1708e4d44ca7b62626c3b27f234e4b86d11ffbbbc16ce","target":"graph","created_at":"2026-07-05T06:59:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.01917/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Human evaluation plays a crucial role in Natural Language Processing (NLP) as it assesses the quality and relevance of developed systems, thereby facilitating their enhancement. However, the absence of widely accepted human evaluation metrics in NLP hampers fair comparisons among different systems and the establishment of universal assessment standards. Through an extensive analysis of existing literature on human evaluation metrics, we identified several gaps in NLP evaluation methodologies. These gaps served as motivation for developing our own hierarchical evaluation framework. The proposed","authors_text":"Iva Bojic, Jessica Chen, Josip Car, Qi Chwen Ong, Shafiq Joty, Si Yuan Chang","cross_cats":["cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-03T09:46:02Z","title":"Hierarchical Evaluation Framework: Best Practices for Human Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.01917","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b864e9066c7a4ba957881a8d9c2c8fc106a2ae1e0844781a2154fb401a7bc4b5","target":"record","created_at":"2026-07-05T06:59:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"00669744024835a16bc96acc60164de2a8d0c8b8634043690f19667c132010ae","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-03T09:46:02Z","title_canon_sha256":"226d9c4334352d6ecc3b05836588b9648bee9b76c2a5079c188b50ffdf53c6fd"},"schema_version":"1.0","source":{"id":"2310.01917","kind":"arxiv","version":2}},"canonical_sha256":"cc0c24996652c0b6283f2ea2cd4727bb67cf46d7ddd5a17328c7e9e3f78a0aac","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cc0c24996652c0b6283f2ea2cd4727bb67cf46d7ddd5a17328c7e9e3f78a0aac","first_computed_at":"2026-07-05T06:59:54.353975Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:59:54.353975Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BJbitDtzTbpte4iWbV3JG9S6UJKLyCNQ9q6SHKE/iiqBFdhzI88AdO8egg1TEN2gve/eF6bU7vGFqkqQeB6MDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:59:54.354442Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.01917","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b864e9066c7a4ba957881a8d9c2c8fc106a2ae1e0844781a2154fb401a7bc4b5","sha256:18f1ea8c7d8dd7d03fb1708e4d44ca7b62626c3b27f234e4b86d11ffbbbc16ce"],"state_sha256":"4062c5a1845ec71e97e0892384f406f1598db9350934011aa67fd4899fd43556"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+dfPAJHpp3HQGvgQu//UCfaagvLO/j+pkT9SdRkyd4Dw5NbohAy/ARBy8UdFnc11EqfJ7/yTVnY0+IMFzR/ODw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T05:04:04.589497Z","bundle_sha256":"59836d943b50ceb18d3126865e9b69716fa38292407a9dea34ca3a66f813fac7"}}