{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:JJUYGUOSA2UUZIV4YMZICL6FHO","short_pith_number":"pith:JJUYGUOS","canonical_record":{"source":{"id":"2206.11249","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-22T17:52:30Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"eed34d151946a72ca40575c5a1e3b1f247f64e1667a7aca264bfcf97fd6507e3","abstract_canon_sha256":"7c5f341a8f069517a170a9e008ee12eaa2f584ec83426489f8bb641edd0567e6"},"schema_version":"1.0"},"canonical_sha256":"4a698351d206a94ca2bcc332812fc53ba0fe6e6fa4bb2b34e8011aecbdb15ecd","source":{"kind":"arxiv","id":"2206.11249","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.11249","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"arxiv_version","alias_value":"2206.11249v3","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.11249","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_12","alias_value":"JJUYGUOSA2UU","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_16","alias_value":"JJUYGUOSA2UUZIV4","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_8","alias_value":"JJUYGUOS","created_at":"2026-07-05T04:34:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:JJUYGUOSA2UUZIV4YMZICL6FHO","target":"record","payload":{"canonical_record":{"source":{"id":"2206.11249","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-22T17:52:30Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"eed34d151946a72ca40575c5a1e3b1f247f64e1667a7aca264bfcf97fd6507e3","abstract_canon_sha256":"7c5f341a8f069517a170a9e008ee12eaa2f584ec83426489f8bb641edd0567e6"},"schema_version":"1.0"},"canonical_sha256":"4a698351d206a94ca2bcc332812fc53ba0fe6e6fa4bb2b34e8011aecbdb15ecd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:34:30.515544Z","signature_b64":"4YZLkzz4GbrArP8uhlGCxDBIJtSvODCUrZJzlehLaKZOix2bbiptCqXcwjVVLhWD+CnkSxkY/Qq3pFmzykE4AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4a698351d206a94ca2bcc332812fc53ba0fe6e6fa4bb2b34e8011aecbdb15ecd","last_reissued_at":"2026-07-05T04:34:30.515079Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:34:30.515079Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2206.11249","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:34:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PVqzCJfc+W+SJGxkpDorewgEWh7Ko7PbZKsmBEQTVg3aL8U26IiQLOJPzBNrlZhC+sO7RxS7m2INGKFV/Ns+AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-05T14:14:30.678718Z"},"content_sha256":"bbfc520dbd13c519047e4aa9e395f89d4ba730fd945259c43cffbb33bfebd509","schema_version":"1.0","event_id":"sha256:bbfc520dbd13c519047e4aa9e395f89d4ba730fd945259c43cffbb33bfebd509"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:JJUYGUOSA2UUZIV4YMZICL6FHO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Abhik Bhattacharjee, Abinaya Mahendiran, Alexandros Papangelis, Alex Wang, Aman Madaan, Angelina McMillan-Major, Anna Shvets, Ashish Upadhyay, Bingsheng Yao, Bryan Wilie, Chandra Bhagavatula, Chaobin You, Craig Thomson, Cristina Garbacea, Dakuo Wang, Daniel Deutsch, Deyi Xiong, Di Jin, Dimitra Gkatzia, Dragomir Radev, Elizabeth Clark, Esin Durmus, Faisal Ladhak, Filip Ginter, Genta Indra Winata, Hendrik Strobelt, Hiroaki Hayashi, Jekaterina Novikova, Jenna Kanerva, Jenny Chim, Jiawei Zhou, Jo\\~ao Sedoc, Jordan Clive, Joshua Maynez, Juraj Juraska, Kaustubh Dhole, Khyathi Raghavi Chandu, Laura Perez-Beltrachini, Leonardo F. R. Ribeiro, Lewis Tunstall, Li Zhang, Mahima Pushkarna, Mathias Creutz, Michael White, Mihir Sanjay Kale, Moussa Kamal Eddine, Nico Daheim, Nishant Subramani, Ondrej Dusek, Paul Pu Liang, Pawan Sasanka Ammanamanchi, Qi Zhu, Ratish Puduppully, Reno Kriz, Rifat Shahriyar, Ronald Cardenas, Saad Mahamood, Salomey Osei, Samuel Cahyawijaya, Sanja \\v{S}tajner, Sebastian Gehrmann, Sebastien Montella, Shailza, Shailza Jolly, Simon Mille, Tahmid Hasan, Tianhao Shen, Tosin Adewumi, Vikas Raunak, Vipul Raheja, Vitaly Nikolaev, Vivian Tsai, Yacine Jernite, Ying Xu, Yisi Sang, Yixin Liu, Yufang Hou","submitted_at":"2022-06-22T17:52:30Z","abstract_excerpt":"Evaluation in machine learning is usually informed by past choices, for example which datasets or metrics to use. This standardization enables the comparison on equal footing using leaderboards, but the evaluation choices become sub-optimal as better alternatives arise. This problem is especially pertinent in natural language generation which requires ever-improving suites of datasets, metrics, and human evaluation to make definitive claims. To make following best model evaluation practices easier, we introduce GEMv2. The new version of the Generation, Evaluation, and Metrics Benchmark introdu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.11249","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2206.11249/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:34:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WfxC7mh3x+UZSm44+CCoe3Gblq4npdjLWNXipse1gtSkMrGx6EdfMVSRrc5tJuj0p1xWGR5OxR8wU0IlI0XoCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-05T14:14:30.679117Z"},"content_sha256":"09b084a4ce484866e3de069a2f63620db53efa51cedfdac1638f8884753477ae","schema_version":"1.0","event_id":"sha256:09b084a4ce484866e3de069a2f63620db53efa51cedfdac1638f8884753477ae"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/bundle.json","state_url":"https://pith.science/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-05T14:14:30Z","links":{"resolver":"https://pith.science/pith/JJUYGUOSA2UUZIV4YMZICL6FHO","bundle":"https://pith.science/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/bundle.json","state":"https://pith.science/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JJUYGUOSA2UUZIV4YMZICL6FHO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:JJUYGUOSA2UUZIV4YMZICL6FHO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7c5f341a8f069517a170a9e008ee12eaa2f584ec83426489f8bb641edd0567e6","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-22T17:52:30Z","title_canon_sha256":"eed34d151946a72ca40575c5a1e3b1f247f64e1667a7aca264bfcf97fd6507e3"},"schema_version":"1.0","source":{"id":"2206.11249","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.11249","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"arxiv_version","alias_value":"2206.11249v3","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.11249","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_12","alias_value":"JJUYGUOSA2UU","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_16","alias_value":"JJUYGUOSA2UUZIV4","created_at":"2026-07-05T04:34:30Z"},{"alias_kind":"pith_short_8","alias_value":"JJUYGUOS","created_at":"2026-07-05T04:34:30Z"}],"graph_snapshots":[{"event_id":"sha256:09b084a4ce484866e3de069a2f63620db53efa51cedfdac1638f8884753477ae","target":"graph","created_at":"2026-07-05T04:34:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2206.11249/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluation in machine learning is usually informed by past choices, for example which datasets or metrics to use. This standardization enables the comparison on equal footing using leaderboards, but the evaluation choices become sub-optimal as better alternatives arise. This problem is especially pertinent in natural language generation which requires ever-improving suites of datasets, metrics, and human evaluation to make definitive claims. To make following best model evaluation practices easier, we introduce GEMv2. The new version of the Generation, Evaluation, and Metrics Benchmark introdu","authors_text":"Abhik Bhattacharjee, Abinaya Mahendiran, Alexandros Papangelis, Alex Wang, Aman Madaan, Angelina McMillan-Major, Anna Shvets, Ashish Upadhyay, Bingsheng Yao, Bryan Wilie, Chandra Bhagavatula, Chaobin You, Craig Thomson, Cristina Garbacea, Dakuo Wang, Daniel Deutsch, Deyi Xiong, Di Jin, Dimitra Gkatzia, Dragomir Radev, Elizabeth Clark, Esin Durmus, Faisal Ladhak, Filip Ginter, Genta Indra Winata, Hendrik Strobelt, Hiroaki Hayashi, Jekaterina Novikova, Jenna Kanerva, Jenny Chim, Jiawei Zhou, Jo\\~ao Sedoc, Jordan Clive, Joshua Maynez, Juraj Juraska, Kaustubh Dhole, Khyathi Raghavi Chandu, Laura Perez-Beltrachini, Leonardo F. R. Ribeiro, Lewis Tunstall, Li Zhang, Mahima Pushkarna, Mathias Creutz, Michael White, Mihir Sanjay Kale, Moussa Kamal Eddine, Nico Daheim, Nishant Subramani, Ondrej Dusek, Paul Pu Liang, Pawan Sasanka Ammanamanchi, Qi Zhu, Ratish Puduppully, Reno Kriz, Rifat Shahriyar, Ronald Cardenas, Saad Mahamood, Salomey Osei, Samuel Cahyawijaya, Sanja \\v{S}tajner, Sebastian Gehrmann, Sebastien Montella, Shailza, Shailza Jolly, Simon Mille, Tahmid Hasan, Tianhao Shen, Tosin Adewumi, Vikas Raunak, Vipul Raheja, Vitaly Nikolaev, Vivian Tsai, Yacine Jernite, Ying Xu, Yisi Sang, Yixin Liu, Yufang Hou","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-22T17:52:30Z","title":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.11249","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bbfc520dbd13c519047e4aa9e395f89d4ba730fd945259c43cffbb33bfebd509","target":"record","created_at":"2026-07-05T04:34:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7c5f341a8f069517a170a9e008ee12eaa2f584ec83426489f8bb641edd0567e6","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-22T17:52:30Z","title_canon_sha256":"eed34d151946a72ca40575c5a1e3b1f247f64e1667a7aca264bfcf97fd6507e3"},"schema_version":"1.0","source":{"id":"2206.11249","kind":"arxiv","version":3}},"canonical_sha256":"4a698351d206a94ca2bcc332812fc53ba0fe6e6fa4bb2b34e8011aecbdb15ecd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4a698351d206a94ca2bcc332812fc53ba0fe6e6fa4bb2b34e8011aecbdb15ecd","first_computed_at":"2026-07-05T04:34:30.515079Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:34:30.515079Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4YZLkzz4GbrArP8uhlGCxDBIJtSvODCUrZJzlehLaKZOix2bbiptCqXcwjVVLhWD+CnkSxkY/Qq3pFmzykE4AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:34:30.515544Z","signed_message":"canonical_sha256_bytes"},"source_id":"2206.11249","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bbfc520dbd13c519047e4aa9e395f89d4ba730fd945259c43cffbb33bfebd509","sha256:09b084a4ce484866e3de069a2f63620db53efa51cedfdac1638f8884753477ae"],"state_sha256":"204a43059dd21c284297f6fc11bcaa630db143c57ee140003484b30878180646"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hHxVzvZ2N2REdqNTlA/l2mlbMj8etmsDsPo1UtcaDkA3HyAM6JQGMNznSMVbAPCFNq34wXacQPs3Mf2Pn4M5AQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-05T14:14:30.681429Z","bundle_sha256":"ca3547059f9cfdb6efd868097ec4ffdbce04f297863bc653b17f2701ae751517"}}