{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:7MUUBDSUOUT3D7KN7NJLJQXCFU","short_pith_number":"pith:7MUUBDSU","canonical_record":{"source":{"id":"2112.04139","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","cross_cats_sorted":[],"title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14","abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1"},"schema_version":"1.0"},"canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","source":{"kind":"arxiv","id":"2112.04139","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"arxiv_version","alias_value":"2112.04139v2","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_12","alias_value":"7MUUBDSUOUT3","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_16","alias_value":"7MUUBDSUOUT3D7KN","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_8","alias_value":"7MUUBDSU","created_at":"2026-07-05T04:24:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:7MUUBDSUOUT3D7KN7NJLJQXCFU","target":"record","payload":{"canonical_record":{"source":{"id":"2112.04139","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","cross_cats_sorted":[],"title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14","abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1"},"schema_version":"1.0"},"canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:24:37.024819Z","signature_b64":"YalODZJ/9zeEmT+78vNlg+c/4sHzTfzZojqu1t5iONVx7ftEzzOXooyBTD15QXQ4rlT5xPSX5ar4KgiFe7jJAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","last_reissued_at":"2026-07-05T04:24:37.024381Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:24:37.024381Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2112.04139","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:24:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VGq46MZiyO82Nc1lVcDg0/eVuotbB7TinpomT41Nbat5JcHJzYSqZtPrzOaxkNt2tx+YI7nvS2zCqMvV5M3UCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:38:53.369214Z"},"content_sha256":"9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a","schema_version":"1.0","event_id":"sha256:9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:7MUUBDSUOUT3D7KN7NJLJQXCFU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Bidimensional Leaderboards: Generate and Evaluate Language Hand in Hand","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexander R. Fabbri, Jacob Morrison, Jungo Kasai, Keisuke Sakaguchi, Lavinia Dunagan, Noah A. Smith, Ronan Le Bras, Yejin Choi","submitted_at":"2021-12-08T06:34:58Z","abstract_excerpt":"Natural language processing researchers have identified limitations of evaluation methodology for generation tasks, with new questions raised about the validity of automatic metrics and of crowdworker judgments. Meanwhile, efforts to improve generation models tend to depend on simple n-gram overlap metrics (e.g., BLEU, ROUGE). We argue that new advances on models and metrics should each more directly benefit and inform the other. We therefore propose a generalization of leaderboards, bidimensional leaderboards (Billboards), that simultaneously tracks progress in language generation models and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.04139","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2112.04139/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:24:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ea3c0jHETRsqf6NxBMCtGZWPeHwuXFXnrgn8wuzQNWLr4AJMH5EP/MNMCOlUCP66n+MbQHo2prgnEE9IMAF5Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:38:53.369599Z"},"content_sha256":"6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f","schema_version":"1.0","event_id":"sha256:6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/bundle.json","state_url":"https://pith.science/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T19:38:53Z","links":{"resolver":"https://pith.science/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU","bundle":"https://pith.science/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/bundle.json","state":"https://pith.science/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7MUUBDSUOUT3D7KN7NJLJQXCFU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:7MUUBDSUOUT3D7KN7NJLJQXCFU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14"},"schema_version":"1.0","source":{"id":"2112.04139","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"arxiv_version","alias_value":"2112.04139v2","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.04139","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_12","alias_value":"7MUUBDSUOUT3","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_16","alias_value":"7MUUBDSUOUT3D7KN","created_at":"2026-07-05T04:24:37Z"},{"alias_kind":"pith_short_8","alias_value":"7MUUBDSU","created_at":"2026-07-05T04:24:37Z"}],"graph_snapshots":[{"event_id":"sha256:6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f","target":"graph","created_at":"2026-07-05T04:24:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2112.04139/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Natural language processing researchers have identified limitations of evaluation methodology for generation tasks, with new questions raised about the validity of automatic metrics and of crowdworker judgments. Meanwhile, efforts to improve generation models tend to depend on simple n-gram overlap metrics (e.g., BLEU, ROUGE). We argue that new advances on models and metrics should each more directly benefit and inform the other. We therefore propose a generalization of leaderboards, bidimensional leaderboards (Billboards), that simultaneously tracks progress in language generation models and ","authors_text":"Alexander R. Fabbri, Jacob Morrison, Jungo Kasai, Keisuke Sakaguchi, Lavinia Dunagan, Noah A. Smith, Ronan Le Bras, Yejin Choi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title":"Bidimensional Leaderboards: Generate and Evaluate Language Hand in Hand"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.04139","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a","target":"record","created_at":"2026-07-05T04:24:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"60e489482669521e2d1d2faf4f1a832f599bef06a7188987d1d336199aaef2c1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-12-08T06:34:58Z","title_canon_sha256":"5859e290a7dcdff876570241ff81e924a9cd3b0f0258affebcf6f5de4773dd14"},"schema_version":"1.0","source":{"id":"2112.04139","kind":"arxiv","version":2}},"canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fb29408e547527b1fd4dfb52b4c2e22d3e632532f259ce7196355a87888bd1c6","first_computed_at":"2026-07-05T04:24:37.024381Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:24:37.024381Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"YalODZJ/9zeEmT+78vNlg+c/4sHzTfzZojqu1t5iONVx7ftEzzOXooyBTD15QXQ4rlT5xPSX5ar4KgiFe7jJAw==","signature_status":"signed_v1","signed_at":"2026-07-05T04:24:37.024819Z","signed_message":"canonical_sha256_bytes"},"source_id":"2112.04139","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9c81d3fd9171c009ada0d0e3fc704ebf7407dbc6c92e2442a3d698f7d531297a","sha256:6fa1b8c0abffea58aff0a184b45eb65167391f5d001eaa672285f7965891055f"],"state_sha256":"99b0f3a91fd718dd9734cae6febb3f23814eee26672cf692caa8abd39972ac97"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FV28amgtNeTlwoqD4VQhzbmGxZmszMGovKIDkWxX1GW5EBtdWgzUC/dP/7yPKOlwZrx9iwLQv0ZSHA8I9wY/DQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T19:38:53.372290Z","bundle_sha256":"290b7c465ca5847f68b2266b8c70d9de893cff80b0e170917bcf23431e3e9fbb"}}