{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:MHHPNS4LUJKZEQ4CXMGBBN43XK","short_pith_number":"pith:MHHPNS4L","canonical_record":{"source":{"id":"2305.14889","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T08:38:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bd92e195840b2307680aca6a09a41cd716d93d1eaf917a0fd3ebbf8585d22318","abstract_canon_sha256":"d9e4b7f71e15fbf3ea8675f7b28e6b97ed0644d22b136d71885d5e2906a40f5b"},"schema_version":"1.0"},"canonical_sha256":"61cef6cb8ba255924382bb0c10b79bbaa668f2a57c1a9245cbf1bb425d4a4cce","source":{"kind":"arxiv","id":"2305.14889","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14889","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14889v2","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14889","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_12","alias_value":"MHHPNS4LUJKZ","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_16","alias_value":"MHHPNS4LUJKZEQ4C","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_8","alias_value":"MHHPNS4L","created_at":"2026-07-05T07:03:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:MHHPNS4LUJKZEQ4CXMGBBN43XK","target":"record","payload":{"canonical_record":{"source":{"id":"2305.14889","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T08:38:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bd92e195840b2307680aca6a09a41cd716d93d1eaf917a0fd3ebbf8585d22318","abstract_canon_sha256":"d9e4b7f71e15fbf3ea8675f7b28e6b97ed0644d22b136d71885d5e2906a40f5b"},"schema_version":"1.0"},"canonical_sha256":"61cef6cb8ba255924382bb0c10b79bbaa668f2a57c1a9245cbf1bb425d4a4cce","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:03:25.806061Z","signature_b64":"mrYPKRA0wQi1u0u2kfMgmuEA+53OorkSu3hYyH6xOTaYOXxxlkVFktsl4eCQPl0ZlUUzGpwZuML3Vx+HqbvMDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"61cef6cb8ba255924382bb0c10b79bbaa668f2a57c1a9245cbf1bb425d4a4cce","last_reissued_at":"2026-07-05T07:03:25.805581Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:03:25.805581Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.14889","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:03:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WTn5/adarTogrvGmz1n9YFLEnI2Qt/ScNxeBaC2pCB0LrwHICyvIRhtRZpAx1kemQkUn9u0fNj10/bF3JH4RBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T18:19:20.900033Z"},"content_sha256":"59c70699ccdc48a8244f3a1a3673d29deb1188f5354fd6299749b3b3356100ab","schema_version":"1.0","event_id":"sha256:59c70699ccdc48a8244f3a1a3673d29deb1188f5354fd6299749b3b3356100ab"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:MHHPNS4LUJKZEQ4CXMGBBN43XK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Evaluation Metrics: A Framework for Analyzing NLG Evaluation Metrics using Measurement Theory","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Q. Vera Liao, Susu Zhang, Vivian Lai, Ziang Xiao","submitted_at":"2023-05-24T08:38:23Z","abstract_excerpt":"We address a fundamental challenge in Natural Language Generation (NLG) model evaluation -- the design and evaluation of evaluation metrics. Recognizing the limitations of existing automatic metrics and noises from how current human evaluation was conducted, we propose MetricEval, a framework informed by measurement theory, the foundation of educational test design, for conceptualizing and evaluating the reliability and validity of NLG evaluation metrics. The framework formalizes the source of measurement error and offers statistical tools for evaluating evaluation metrics based on empirical d"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14889","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.14889/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:03:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jCsX6hqbEJxifkU3BIWHMg2DNXStaCTfZTiKPHRT/8s/hgBsTwngFt0ddJL3u61j4bPDXs8aWdA3ZVJyaeq9AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T18:19:20.900596Z"},"content_sha256":"cf18754fa591f152229f2ee5f5731e435965f7c0b4050cb74fb82c180188b3ba","schema_version":"1.0","event_id":"sha256:cf18754fa591f152229f2ee5f5731e435965f7c0b4050cb74fb82c180188b3ba"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/bundle.json","state_url":"https://pith.science/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T18:19:20Z","links":{"resolver":"https://pith.science/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK","bundle":"https://pith.science/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/bundle.json","state":"https://pith.science/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MHHPNS4LUJKZEQ4CXMGBBN43XK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:MHHPNS4LUJKZEQ4CXMGBBN43XK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d9e4b7f71e15fbf3ea8675f7b28e6b97ed0644d22b136d71885d5e2906a40f5b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T08:38:23Z","title_canon_sha256":"bd92e195840b2307680aca6a09a41cd716d93d1eaf917a0fd3ebbf8585d22318"},"schema_version":"1.0","source":{"id":"2305.14889","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14889","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14889v2","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14889","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_12","alias_value":"MHHPNS4LUJKZ","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_16","alias_value":"MHHPNS4LUJKZEQ4C","created_at":"2026-07-05T07:03:25Z"},{"alias_kind":"pith_short_8","alias_value":"MHHPNS4L","created_at":"2026-07-05T07:03:25Z"}],"graph_snapshots":[{"event_id":"sha256:cf18754fa591f152229f2ee5f5731e435965f7c0b4050cb74fb82c180188b3ba","target":"graph","created_at":"2026-07-05T07:03:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.14889/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We address a fundamental challenge in Natural Language Generation (NLG) model evaluation -- the design and evaluation of evaluation metrics. Recognizing the limitations of existing automatic metrics and noises from how current human evaluation was conducted, we propose MetricEval, a framework informed by measurement theory, the foundation of educational test design, for conceptualizing and evaluating the reliability and validity of NLG evaluation metrics. The framework formalizes the source of measurement error and offers statistical tools for evaluating evaluation metrics based on empirical d","authors_text":"Q. Vera Liao, Susu Zhang, Vivian Lai, Ziang Xiao","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T08:38:23Z","title":"Evaluating Evaluation Metrics: A Framework for Analyzing NLG Evaluation Metrics using Measurement Theory"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14889","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:59c70699ccdc48a8244f3a1a3673d29deb1188f5354fd6299749b3b3356100ab","target":"record","created_at":"2026-07-05T07:03:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d9e4b7f71e15fbf3ea8675f7b28e6b97ed0644d22b136d71885d5e2906a40f5b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T08:38:23Z","title_canon_sha256":"bd92e195840b2307680aca6a09a41cd716d93d1eaf917a0fd3ebbf8585d22318"},"schema_version":"1.0","source":{"id":"2305.14889","kind":"arxiv","version":2}},"canonical_sha256":"61cef6cb8ba255924382bb0c10b79bbaa668f2a57c1a9245cbf1bb425d4a4cce","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"61cef6cb8ba255924382bb0c10b79bbaa668f2a57c1a9245cbf1bb425d4a4cce","first_computed_at":"2026-07-05T07:03:25.805581Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:03:25.805581Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mrYPKRA0wQi1u0u2kfMgmuEA+53OorkSu3hYyH6xOTaYOXxxlkVFktsl4eCQPl0ZlUUzGpwZuML3Vx+HqbvMDw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:03:25.806061Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.14889","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:59c70699ccdc48a8244f3a1a3673d29deb1188f5354fd6299749b3b3356100ab","sha256:cf18754fa591f152229f2ee5f5731e435965f7c0b4050cb74fb82c180188b3ba"],"state_sha256":"d235783f4046dd00cd6eddb90a92672f83212c4aca1172fa2153ef2bde240f84"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QxQ69OWNPz29V9HXW5ETk2qSQfniDB9UsDFlm9xSq75DT9xxHm3cFNAAA7IHgq/RP14a3i4z+PjIv4Z7P4pNCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T18:19:20.906719Z","bundle_sha256":"34cd1b975eb2fef4381251f589c2ae305ead0954d382851daf9283632709f8c3"}}