{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:6AVH65TCD34KM6OKQ3B5EGFIUE","short_pith_number":"pith:6AVH65TC","canonical_record":{"source":{"id":"2105.08920","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-19T04:45:07Z","cross_cats_sorted":[],"title_canon_sha256":"0dc0508f26410d93ab6582231587a314afaf4e902a4ed9e5dfe18bf15fc7574e","abstract_canon_sha256":"4826d01b97ca55620a90c829829b4347702fd98aca7b29e7ba3a2910a0f97d06"},"schema_version":"1.0"},"canonical_sha256":"f02a7f76621ef8a679ca86c3d218a8a139219214d3ad5685c1f9e6c350a5b78c","source":{"kind":"arxiv","id":"2105.08920","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.08920","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"arxiv_version","alias_value":"2105.08920v1","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.08920","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_12","alias_value":"6AVH65TCD34K","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_16","alias_value":"6AVH65TCD34KM6OK","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_8","alias_value":"6AVH65TC","created_at":"2026-07-05T02:41:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:6AVH65TCD34KM6OKQ3B5EGFIUE","target":"record","payload":{"canonical_record":{"source":{"id":"2105.08920","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-19T04:45:07Z","cross_cats_sorted":[],"title_canon_sha256":"0dc0508f26410d93ab6582231587a314afaf4e902a4ed9e5dfe18bf15fc7574e","abstract_canon_sha256":"4826d01b97ca55620a90c829829b4347702fd98aca7b29e7ba3a2910a0f97d06"},"schema_version":"1.0"},"canonical_sha256":"f02a7f76621ef8a679ca86c3d218a8a139219214d3ad5685c1f9e6c350a5b78c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:41:35.188634Z","signature_b64":"PPiEb6x2FrIGSbqvOb1ul5V/NAmtZCQLfPrg1jwx29+QS582Rvg8ilIZfei72+rbihrn1TbrswSr8hgFyJQgCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f02a7f76621ef8a679ca86c3d218a8a139219214d3ad5685c1f9e6c350a5b78c","last_reissued_at":"2026-07-05T02:41:35.188178Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:41:35.188178Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2105.08920","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:41:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L8RzQpufDkm/0Wb7UKfCzvsB9aINNyLUE7My2RUY4yRCP1EKvB+RthVsu5BpE6EvQRNlIZXJoKhKFfW5k7eRBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:00:54.951753Z"},"content_sha256":"72ee3fee76ad7e5a6ba0d6f847f7cde7f84fda51bb939b7e34d4f4e1094c523c","schema_version":"1.0","event_id":"sha256:72ee3fee76ad7e5a6ba0d6f847f7cde7f84fda51bb939b7e34d4f4e1094c523c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:6AVH65TCD34KM6OKQ3B5EGFIUE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"OpenMEVA: A Benchmark for Evaluating Open-ended Story Generation Metrics","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Changjie Fan, Jian Guan, Minlie Huang, Wenbiao Ding, Xiaoxi Mao, Zhexin Zhang, Zhuoer Feng, Zitao Liu","submitted_at":"2021-05-19T04:45:07Z","abstract_excerpt":"Automatic metrics are essential for developing natural language generation (NLG) models, particularly for open-ended language generation tasks such as story generation. However, existing automatic metrics are observed to correlate poorly with human evaluation. The lack of standardized benchmark datasets makes it difficult to fully evaluate the capabilities of a metric and fairly compare different metrics. Therefore, we propose OpenMEVA, a benchmark for evaluating open-ended story generation metrics. OpenMEVA provides a comprehensive test suite to assess the capabilities of metrics, including ("},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.08920","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.08920/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:41:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"myLCP95BbBppRORFGleOYklf6xzmnmUtFp5QqR5MpcyqPuIUOqwSQQVY+jyHEjUNnQXhMdCA1OPjPs4JEWuYCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:00:54.952383Z"},"content_sha256":"c28cca0c147eb05cd47ce1decd2e081abcc0086b5bac909de5651dd5d23f28e6","schema_version":"1.0","event_id":"sha256:c28cca0c147eb05cd47ce1decd2e081abcc0086b5bac909de5651dd5d23f28e6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/bundle.json","state_url":"https://pith.science/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:00:54Z","links":{"resolver":"https://pith.science/pith/6AVH65TCD34KM6OKQ3B5EGFIUE","bundle":"https://pith.science/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/bundle.json","state":"https://pith.science/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6AVH65TCD34KM6OKQ3B5EGFIUE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:6AVH65TCD34KM6OKQ3B5EGFIUE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4826d01b97ca55620a90c829829b4347702fd98aca7b29e7ba3a2910a0f97d06","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-19T04:45:07Z","title_canon_sha256":"0dc0508f26410d93ab6582231587a314afaf4e902a4ed9e5dfe18bf15fc7574e"},"schema_version":"1.0","source":{"id":"2105.08920","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.08920","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"arxiv_version","alias_value":"2105.08920v1","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.08920","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_12","alias_value":"6AVH65TCD34K","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_16","alias_value":"6AVH65TCD34KM6OK","created_at":"2026-07-05T02:41:35Z"},{"alias_kind":"pith_short_8","alias_value":"6AVH65TC","created_at":"2026-07-05T02:41:35Z"}],"graph_snapshots":[{"event_id":"sha256:c28cca0c147eb05cd47ce1decd2e081abcc0086b5bac909de5651dd5d23f28e6","target":"graph","created_at":"2026-07-05T02:41:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2105.08920/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic metrics are essential for developing natural language generation (NLG) models, particularly for open-ended language generation tasks such as story generation. However, existing automatic metrics are observed to correlate poorly with human evaluation. The lack of standardized benchmark datasets makes it difficult to fully evaluate the capabilities of a metric and fairly compare different metrics. Therefore, we propose OpenMEVA, a benchmark for evaluating open-ended story generation metrics. OpenMEVA provides a comprehensive test suite to assess the capabilities of metrics, including (","authors_text":"Changjie Fan, Jian Guan, Minlie Huang, Wenbiao Ding, Xiaoxi Mao, Zhexin Zhang, Zhuoer Feng, Zitao Liu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-19T04:45:07Z","title":"OpenMEVA: A Benchmark for Evaluating Open-ended Story Generation Metrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.08920","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:72ee3fee76ad7e5a6ba0d6f847f7cde7f84fda51bb939b7e34d4f4e1094c523c","target":"record","created_at":"2026-07-05T02:41:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4826d01b97ca55620a90c829829b4347702fd98aca7b29e7ba3a2910a0f97d06","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-19T04:45:07Z","title_canon_sha256":"0dc0508f26410d93ab6582231587a314afaf4e902a4ed9e5dfe18bf15fc7574e"},"schema_version":"1.0","source":{"id":"2105.08920","kind":"arxiv","version":1}},"canonical_sha256":"f02a7f76621ef8a679ca86c3d218a8a139219214d3ad5685c1f9e6c350a5b78c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f02a7f76621ef8a679ca86c3d218a8a139219214d3ad5685c1f9e6c350a5b78c","first_computed_at":"2026-07-05T02:41:35.188178Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:41:35.188178Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"PPiEb6x2FrIGSbqvOb1ul5V/NAmtZCQLfPrg1jwx29+QS582Rvg8ilIZfei72+rbihrn1TbrswSr8hgFyJQgCg==","signature_status":"signed_v1","signed_at":"2026-07-05T02:41:35.188634Z","signed_message":"canonical_sha256_bytes"},"source_id":"2105.08920","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:72ee3fee76ad7e5a6ba0d6f847f7cde7f84fda51bb939b7e34d4f4e1094c523c","sha256:c28cca0c147eb05cd47ce1decd2e081abcc0086b5bac909de5651dd5d23f28e6"],"state_sha256":"729a5993f921fea42fddbf7c4e6959913876e4ea451b00d0d1654e44a8d81a84"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QEaxTg1UfFxHtdlwLfJFbEPaDEMdRBHo+mZTIiNNNBScKY8D33+b8xGz6Br/dPejfJT2DaeEvQz0nsRg/o4jCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:00:54.958617Z","bundle_sha256":"a3b1a4450be00264fb5886b0d8c19d3b52f91460f403a71af526512b86c842c0"}}