{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:WVGSSID5YVU7CI3XDMFEZ4ZXNU","short_pith_number":"pith:WVGSSID5","canonical_record":{"source":{"id":"2009.13888","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-29T09:25:31Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5ea30758bb56f2811310a0bd261686ede4720f7367565aa13f0ed0b28f469b9b","abstract_canon_sha256":"ad4d644131ba58070a5e7155f9998a83f9ff96c2a165090a5075158b968ac991"},"schema_version":"1.0"},"canonical_sha256":"b54d29207dc569f123771b0a4cf3376d282c5081f26a45cab9e110b7c1c5f767","source":{"kind":"arxiv","id":"2009.13888","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2009.13888","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"arxiv_version","alias_value":"2009.13888v4","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.13888","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_12","alias_value":"WVGSSID5YVU7","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_16","alias_value":"WVGSSID5YVU7CI3X","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_8","alias_value":"WVGSSID5","created_at":"2026-07-05T02:19:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:WVGSSID5YVU7CI3XDMFEZ4ZXNU","target":"record","payload":{"canonical_record":{"source":{"id":"2009.13888","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-29T09:25:31Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5ea30758bb56f2811310a0bd261686ede4720f7367565aa13f0ed0b28f469b9b","abstract_canon_sha256":"ad4d644131ba58070a5e7155f9998a83f9ff96c2a165090a5075158b968ac991"},"schema_version":"1.0"},"canonical_sha256":"b54d29207dc569f123771b0a4cf3376d282c5081f26a45cab9e110b7c1c5f767","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:19:56.155852Z","signature_b64":"ic/4F6NTPYe8e7SYvEs/+SUVG3XL3gqVQ5Jo3Dt2pxAG7Anf51x+FwOcQCZj3c3mnET+M6hTtSEv6ExZJydNDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b54d29207dc569f123771b0a4cf3376d282c5081f26a45cab9e110b7c1c5f767","last_reissued_at":"2026-07-05T02:19:56.155352Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:19:56.155352Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2009.13888","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:19:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CVXwkZ0v9HYYy9luYdnehmsJzVG6RFiLgdjTrEVMVrP3RoMm+YPVx3BZ3jcqLve0P4rkUC0BuRjw1y5cBlEoCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T20:56:33.472948Z"},"content_sha256":"a4bd1d5199b6c1f3b6ce2956143a1004f0e914468aaa6f600980cf178570cf38","schema_version":"1.0","event_id":"sha256:a4bd1d5199b6c1f3b6ce2956143a1004f0e914468aaa6f600980cf178570cf38"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:WVGSSID5YVU7CI3XDMFEZ4ZXNU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Utility is in the Eye of the User: A Critique of NLP Leaderboards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Dan Jurafsky, Kawin Ethayarajh","submitted_at":"2020-09-29T09:25:31Z","abstract_excerpt":"Benchmarks such as GLUE have helped drive advances in NLP by incentivizing the creation of more accurate models. While this leaderboard paradigm has been remarkably successful, a historical focus on performance-based evaluation has been at the expense of other qualities that the NLP community values in models, such as compactness, fairness, and energy efficiency. In this opinion paper, we study the divergence between what is incentivized by leaderboards and what is useful in practice through the lens of microeconomic theory. We frame both the leaderboard and NLP practitioners as consumers and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.13888","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2009.13888/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:19:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9mav2FtBlbzm/HIR1YLQpDvx1qeLnRnaycoa4ZBRQTBryTT1KuflOVVqTA+e5/Q+lEQ0yYAIwETEDKPBYMldBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T20:56:33.473477Z"},"content_sha256":"dc6b866e34dfdf0c93934c7c9429c9a507b2060567a86fabab00dbaef230e91a","schema_version":"1.0","event_id":"sha256:dc6b866e34dfdf0c93934c7c9429c9a507b2060567a86fabab00dbaef230e91a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/bundle.json","state_url":"https://pith.science/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T20:56:33Z","links":{"resolver":"https://pith.science/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU","bundle":"https://pith.science/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/bundle.json","state":"https://pith.science/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WVGSSID5YVU7CI3XDMFEZ4ZXNU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:WVGSSID5YVU7CI3XDMFEZ4ZXNU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ad4d644131ba58070a5e7155f9998a83f9ff96c2a165090a5075158b968ac991","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-29T09:25:31Z","title_canon_sha256":"5ea30758bb56f2811310a0bd261686ede4720f7367565aa13f0ed0b28f469b9b"},"schema_version":"1.0","source":{"id":"2009.13888","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2009.13888","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"arxiv_version","alias_value":"2009.13888v4","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.13888","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_12","alias_value":"WVGSSID5YVU7","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_16","alias_value":"WVGSSID5YVU7CI3X","created_at":"2026-07-05T02:19:56Z"},{"alias_kind":"pith_short_8","alias_value":"WVGSSID5","created_at":"2026-07-05T02:19:56Z"}],"graph_snapshots":[{"event_id":"sha256:dc6b866e34dfdf0c93934c7c9429c9a507b2060567a86fabab00dbaef230e91a","target":"graph","created_at":"2026-07-05T02:19:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2009.13888/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Benchmarks such as GLUE have helped drive advances in NLP by incentivizing the creation of more accurate models. While this leaderboard paradigm has been remarkably successful, a historical focus on performance-based evaluation has been at the expense of other qualities that the NLP community values in models, such as compactness, fairness, and energy efficiency. In this opinion paper, we study the divergence between what is incentivized by leaderboards and what is useful in practice through the lens of microeconomic theory. We frame both the leaderboard and NLP practitioners as consumers and ","authors_text":"Dan Jurafsky, Kawin Ethayarajh","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-29T09:25:31Z","title":"Utility is in the Eye of the User: A Critique of NLP Leaderboards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.13888","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a4bd1d5199b6c1f3b6ce2956143a1004f0e914468aaa6f600980cf178570cf38","target":"record","created_at":"2026-07-05T02:19:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ad4d644131ba58070a5e7155f9998a83f9ff96c2a165090a5075158b968ac991","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-09-29T09:25:31Z","title_canon_sha256":"5ea30758bb56f2811310a0bd261686ede4720f7367565aa13f0ed0b28f469b9b"},"schema_version":"1.0","source":{"id":"2009.13888","kind":"arxiv","version":4}},"canonical_sha256":"b54d29207dc569f123771b0a4cf3376d282c5081f26a45cab9e110b7c1c5f767","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b54d29207dc569f123771b0a4cf3376d282c5081f26a45cab9e110b7c1c5f767","first_computed_at":"2026-07-05T02:19:56.155352Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:19:56.155352Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ic/4F6NTPYe8e7SYvEs/+SUVG3XL3gqVQ5Jo3Dt2pxAG7Anf51x+FwOcQCZj3c3mnET+M6hTtSEv6ExZJydNDw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:19:56.155852Z","signed_message":"canonical_sha256_bytes"},"source_id":"2009.13888","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a4bd1d5199b6c1f3b6ce2956143a1004f0e914468aaa6f600980cf178570cf38","sha256:dc6b866e34dfdf0c93934c7c9429c9a507b2060567a86fabab00dbaef230e91a"],"state_sha256":"c6ae3ac8e30e152e167f8fb0d5ba890cb4848d02677ac8fd21a4d7369f7f5888"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"V/8e7jn2u7bPGeI9SVmMtQWlD6l6Bie9/qPNEKNt/Ksb9L3LeudR/TLWaAS3wa5LF+U5cHhy23ThAMPCn/kMBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T20:56:33.478666Z","bundle_sha256":"d2fbd45cf4cdd6842aa2c2333cd2f6fe9e560f35427441007809c8859b066cf3"}}