{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:HIQY6G6FFAB67JREQHOLE2WUVN","short_pith_number":"pith:HIQY6G6F","canonical_record":{"source":{"id":"2211.00922","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-02T07:12:23Z","cross_cats_sorted":[],"title_canon_sha256":"7fad36d51af57afa253ef87aa52f3ab68448c6c0d92522b07e9bc79297d66b1b","abstract_canon_sha256":"f22a4cd1c655b31635d2689ee04a0ef127edf7d144e14d92c9b9b2eab116758a"},"schema_version":"1.0"},"canonical_sha256":"3a218f1bc52803efa62481dcb26ad4ab62c0ffbf3c737585e4805d7c68084ec0","source":{"kind":"arxiv","id":"2211.00922","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.00922","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"arxiv_version","alias_value":"2211.00922v1","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.00922","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_12","alias_value":"HIQY6G6FFAB6","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_16","alias_value":"HIQY6G6FFAB67JRE","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_8","alias_value":"HIQY6G6F","created_at":"2026-07-05T05:12:36Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:HIQY6G6FFAB67JREQHOLE2WUVN","target":"record","payload":{"canonical_record":{"source":{"id":"2211.00922","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-02T07:12:23Z","cross_cats_sorted":[],"title_canon_sha256":"7fad36d51af57afa253ef87aa52f3ab68448c6c0d92522b07e9bc79297d66b1b","abstract_canon_sha256":"f22a4cd1c655b31635d2689ee04a0ef127edf7d144e14d92c9b9b2eab116758a"},"schema_version":"1.0"},"canonical_sha256":"3a218f1bc52803efa62481dcb26ad4ab62c0ffbf3c737585e4805d7c68084ec0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:12:36.633943Z","signature_b64":"Ro+hM/3HxLbuYLcZF0vw1UclpdznlZ/kfJ6UNUKfRXJQb1duNiNYHf6tcf98HF42+zvcK7XQgi/CneGnqH45Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3a218f1bc52803efa62481dcb26ad4ab62c0ffbf3c737585e4805d7c68084ec0","last_reissued_at":"2026-07-05T05:12:36.633532Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:12:36.633532Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2211.00922","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sYhlAfsRmqOqjhBUXFM11/ArSIBF2pO2Rm+yl37mM6IXC/GRfk51vPeGqrH77MqOu6fvCaHg244yIx9Gc9m2CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T16:50:50.947314Z"},"content_sha256":"6494dd2fc9bbf21f08e2543aca70a74f84b38b9d25ba494aeebb19fd3561cbbd","schema_version":"1.0","event_id":"sha256:6494dd2fc9bbf21f08e2543aca70a74f84b38b9d25ba494aeebb19fd3561cbbd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:HIQY6G6FFAB67JREQHOLE2WUVN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Dialect-robust Evaluation of Generated Text","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aditya Siddhant, Dan Garrette, Elizabeth Clark, Jacob Eisenstein, Jiao Sun, Sebastian Gehrmann, Thibault Sellam, Timothy Dozat, Tu Vu","submitted_at":"2022-11-02T07:12:23Z","abstract_excerpt":"Evaluation metrics that are not robust to dialect variation make it impossible to tell how well systems perform for many groups of users, and can even penalize systems for producing text in lower-resource dialects. However, currently, there exists no way to quantify how metrics respond to change in the dialect of a generated utterance. We thus formalize dialect robustness and dialect awareness as goals for NLG evaluation metrics. We introduce a suite of methods and corresponding statistical tests one can use to assess metrics in light of the two goals. Applying the suite to current state-of-th"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.00922","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.00922/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8UQGtYYYt/EpHXnvLXEvHjcEZnk1LabEjBOt6aSA5L27Om/s7dhBNL7QYI5YZaOqlINHvzqgc3WHvMg6rzVvCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T16:50:50.947597Z"},"content_sha256":"7a0de1766db67be77abcc77703c35494be9fb0284afcb387ccd4bfc8ad5973f5","schema_version":"1.0","event_id":"sha256:7a0de1766db67be77abcc77703c35494be9fb0284afcb387ccd4bfc8ad5973f5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HIQY6G6FFAB67JREQHOLE2WUVN/bundle.json","state_url":"https://pith.science/pith/HIQY6G6FFAB67JREQHOLE2WUVN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HIQY6G6FFAB67JREQHOLE2WUVN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-31T16:50:50Z","links":{"resolver":"https://pith.science/pith/HIQY6G6FFAB67JREQHOLE2WUVN","bundle":"https://pith.science/pith/HIQY6G6FFAB67JREQHOLE2WUVN/bundle.json","state":"https://pith.science/pith/HIQY6G6FFAB67JREQHOLE2WUVN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HIQY6G6FFAB67JREQHOLE2WUVN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:HIQY6G6FFAB67JREQHOLE2WUVN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f22a4cd1c655b31635d2689ee04a0ef127edf7d144e14d92c9b9b2eab116758a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-02T07:12:23Z","title_canon_sha256":"7fad36d51af57afa253ef87aa52f3ab68448c6c0d92522b07e9bc79297d66b1b"},"schema_version":"1.0","source":{"id":"2211.00922","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.00922","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"arxiv_version","alias_value":"2211.00922v1","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.00922","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_12","alias_value":"HIQY6G6FFAB6","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_16","alias_value":"HIQY6G6FFAB67JRE","created_at":"2026-07-05T05:12:36Z"},{"alias_kind":"pith_short_8","alias_value":"HIQY6G6F","created_at":"2026-07-05T05:12:36Z"}],"graph_snapshots":[{"event_id":"sha256:7a0de1766db67be77abcc77703c35494be9fb0284afcb387ccd4bfc8ad5973f5","target":"graph","created_at":"2026-07-05T05:12:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2211.00922/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluation metrics that are not robust to dialect variation make it impossible to tell how well systems perform for many groups of users, and can even penalize systems for producing text in lower-resource dialects. However, currently, there exists no way to quantify how metrics respond to change in the dialect of a generated utterance. We thus formalize dialect robustness and dialect awareness as goals for NLG evaluation metrics. We introduce a suite of methods and corresponding statistical tests one can use to assess metrics in light of the two goals. Applying the suite to current state-of-th","authors_text":"Aditya Siddhant, Dan Garrette, Elizabeth Clark, Jacob Eisenstein, Jiao Sun, Sebastian Gehrmann, Thibault Sellam, Timothy Dozat, Tu Vu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-02T07:12:23Z","title":"Dialect-robust Evaluation of Generated Text"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.00922","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6494dd2fc9bbf21f08e2543aca70a74f84b38b9d25ba494aeebb19fd3561cbbd","target":"record","created_at":"2026-07-05T05:12:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f22a4cd1c655b31635d2689ee04a0ef127edf7d144e14d92c9b9b2eab116758a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-02T07:12:23Z","title_canon_sha256":"7fad36d51af57afa253ef87aa52f3ab68448c6c0d92522b07e9bc79297d66b1b"},"schema_version":"1.0","source":{"id":"2211.00922","kind":"arxiv","version":1}},"canonical_sha256":"3a218f1bc52803efa62481dcb26ad4ab62c0ffbf3c737585e4805d7c68084ec0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3a218f1bc52803efa62481dcb26ad4ab62c0ffbf3c737585e4805d7c68084ec0","first_computed_at":"2026-07-05T05:12:36.633532Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:12:36.633532Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Ro+hM/3HxLbuYLcZF0vw1UclpdznlZ/kfJ6UNUKfRXJQb1duNiNYHf6tcf98HF42+zvcK7XQgi/CneGnqH45Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:12:36.633943Z","signed_message":"canonical_sha256_bytes"},"source_id":"2211.00922","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6494dd2fc9bbf21f08e2543aca70a74f84b38b9d25ba494aeebb19fd3561cbbd","sha256:7a0de1766db67be77abcc77703c35494be9fb0284afcb387ccd4bfc8ad5973f5"],"state_sha256":"21d935dd595429897bc4c77253a59fd88b61af767eac5b367c896a7ce79408ad"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e9mwThomIrglLhO5YFSnAK6q00W3H3r9aRnFejbTCPEmjh7gMRQfRvC0bmzH52XUxHwU6tNJbF//RflwmhP/CQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-31T16:50:50.950912Z","bundle_sha256":"c1d04bb3f33c998abae3dfc5f43220d5897ca47b77e0d44b91c6785d908d5abc"}}