{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:7MUUBDSUOUT3D7KN7NJLJQXCFU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14"},"schema_version":"1.0","source":{"id":"2112.04139","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"arxiv_version","alias_value":"2112.04139v2","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_12","alias_value":"7MUUBDSUOUT3","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_16","alias_value":"7MUUBDSUOUT3D7KN","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_8","alias_value":"7MUUBDSU","created_at":"2026-07-05T04:24:37Z"}],"graph_snapshots":[{"event_id":"sha256:6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f","target":"graph","created_at":"2026-07-05T04:24:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2112.04139/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Natural language processing researchers have identified limitations of evaluation methodology for generation tasks, with new questions raised about the validity of automatic metrics and of crowdworker judgments. Meanwhile, efforts to improve generation models tend to depend on simple n-gram overlap metrics (e.g., BLEU, ROUGE). We argue that new advances on models and metrics should each more directly benefit and inform the other. We therefore propose a generalization of leaderboards, bidimensional leaderboards (Billboards), that simultaneously tracks progress in language generation models and ","authors_text":"Alexander R. Fabbri, Jacob Morrison, Jungo Kasai, Keisuke Sakaguchi, Lavinia Dunagan, Noah A. Smith, Ronan Le Bras, Yejin Choi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title":"Bidimensional Leaderboards: Generate and Evaluate Language Hand in Hand"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.04139","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a","target":"record","created_at":"2026-07-05T04:24:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14"},"schema_version":"1.0","source":{"id":"2112.04139","kind":"arxiv","version":2}},"canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","first_computed_at":"2026-07-05T04:24:37.024381Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:24:37.024381Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"YalODZJ/9zeEmT+78vNlg+c/4sHzTfzZojqu1t5iONVx7ftEzzOXooyBTD15QXQ4rlT5xPSX5ar4KgiFe7jJAw==","signature_status":"signed_v1","signed_at":"2026-07-05T04:24:37.024819Z","signed_message":"canonical_sha256_bytes"},"source_id":"2112.04139","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a","sha256:6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f"],"state_sha256":"99b0f3a91fd718dd9734cae6febb3f23814eee26672cf692caa8abd39972ac97"}