{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:5JAIX3ZMLKC3V6POGI6BVPAJ3D","short_pith_number":"pith:5JAIX3ZM","canonical_record":{"source":{"id":"2104.14337","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-07T17:49:17Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a60eebbdeedb78e3a6421b8e70cdd26279b1dfe4e7c8b185b7095c20d541680a","abstract_canon_sha256":"91e1e43c0e7aac90b20ca2651ee973a889c85830dd1dc64403a703c0a08d561b"},"schema_version":"1.0"},"canonical_sha256":"ea408bef2c5a85baf9ee323c1abc09d8d989db6c26e13140388a869824caba15","source":{"kind":"arxiv","id":"2104.14337","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.14337","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"arxiv_version","alias_value":"2104.14337v1","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.14337","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_12","alias_value":"5JAIX3ZMLKC3","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_16","alias_value":"5JAIX3ZMLKC3V6PO","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_8","alias_value":"5JAIX3ZM","created_at":"2026-07-05T02:36:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:5JAIX3ZMLKC3V6POGI6BVPAJ3D","target":"record","payload":{"canonical_record":{"source":{"id":"2104.14337","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-07T17:49:17Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a60eebbdeedb78e3a6421b8e70cdd26279b1dfe4e7c8b185b7095c20d541680a","abstract_canon_sha256":"91e1e43c0e7aac90b20ca2651ee973a889c85830dd1dc64403a703c0a08d561b"},"schema_version":"1.0"},"canonical_sha256":"ea408bef2c5a85baf9ee323c1abc09d8d989db6c26e13140388a869824caba15","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:36:15.475983Z","signature_b64":"Jjfhv6cY6Qr/0oQD3VgJ+GBaoT85OB2bMk1UIosz3ALvyadWS6rk3K3xJSOsu0D1x6uWLx42mf4av1RGKsjeAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ea408bef2c5a85baf9ee323c1abc09d8d989db6c26e13140388a869824caba15","last_reissued_at":"2026-07-05T02:36:15.475559Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:36:15.475559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.14337","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:36:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YvsId3eZmQomrAp5OT1z5WQ08GhLlSLIiyxC7jCd7VKQXKDk3Hkv8msceRucPEqnxpRd1csz2PErxMrHUNS9AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T17:21:30.639693Z"},"content_sha256":"695a9463b81644ca5f8cf0f7b8351915718e91373a568e7d08c636a6f754fdef","schema_version":"1.0","event_id":"sha256:695a9463b81644ca5f8cf0f7b8351915718e91373a568e7d08c636a6f754fdef"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:5JAIX3ZMLKC3V6POGI6BVPAJ3D","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Dynabench: Rethinking Benchmarking in NLP","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Adina Williams, Amanpreet Singh, Atticus Geiger, Bertie Vidgen, Christopher Potts, Divyansh Kaushik, Douwe Kiela, Grusha Prasad, Max Bartolo, Mohit Bansal, Pontus Stenetorp, Pratik Ringshia, Robin Jia, Sebastian Riedel, Tristan Thrush, Yixin Nie, Zeerak Waseem, Zhengxuan Wu, Zhiyi Ma","submitted_at":"2021-04-07T17:49:17Z","abstract_excerpt":"We introduce Dynabench, an open-source platform for dynamic dataset creation and model benchmarking. Dynabench runs in a web browser and supports human-and-model-in-the-loop dataset creation: annotators seek to create examples that a target model will misclassify, but that another person will not. In this paper, we argue that Dynabench addresses a critical need in our community: contemporary models quickly achieve outstanding performance on benchmark tasks but nonetheless fail on simple challenge examples and falter in real-world scenarios. With Dynabench, dataset creation, model development, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.14337","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.14337/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:36:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Kumj/BQEd7hTHll0YE7M9JM8C7zuZPaa4rFTApYANymHV2dUotbDNZo6ETvTpa4pMiO6jZ+uK51jMWIFHlylCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T17:21:30.640209Z"},"content_sha256":"84833472e757e990bd8d0485ca30b02d3e4e284c2cef7bf73bf7d74caef09910","schema_version":"1.0","event_id":"sha256:84833472e757e990bd8d0485ca30b02d3e4e284c2cef7bf73bf7d74caef09910"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/bundle.json","state_url":"https://pith.science/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T17:21:30Z","links":{"resolver":"https://pith.science/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D","bundle":"https://pith.science/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/bundle.json","state":"https://pith.science/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5JAIX3ZMLKC3V6POGI6BVPAJ3D/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:5JAIX3ZMLKC3V6POGI6BVPAJ3D","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"91e1e43c0e7aac90b20ca2651ee973a889c85830dd1dc64403a703c0a08d561b","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-07T17:49:17Z","title_canon_sha256":"a60eebbdeedb78e3a6421b8e70cdd26279b1dfe4e7c8b185b7095c20d541680a"},"schema_version":"1.0","source":{"id":"2104.14337","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.14337","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"arxiv_version","alias_value":"2104.14337v1","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.14337","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_12","alias_value":"5JAIX3ZMLKC3","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_16","alias_value":"5JAIX3ZMLKC3V6PO","created_at":"2026-07-05T02:36:15Z"},{"alias_kind":"pith_short_8","alias_value":"5JAIX3ZM","created_at":"2026-07-05T02:36:15Z"}],"graph_snapshots":[{"event_id":"sha256:84833472e757e990bd8d0485ca30b02d3e4e284c2cef7bf73bf7d74caef09910","target":"graph","created_at":"2026-07-05T02:36:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.14337/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We introduce Dynabench, an open-source platform for dynamic dataset creation and model benchmarking. Dynabench runs in a web browser and supports human-and-model-in-the-loop dataset creation: annotators seek to create examples that a target model will misclassify, but that another person will not. In this paper, we argue that Dynabench addresses a critical need in our community: contemporary models quickly achieve outstanding performance on benchmark tasks but nonetheless fail on simple challenge examples and falter in real-world scenarios. With Dynabench, dataset creation, model development, ","authors_text":"Adina Williams, Amanpreet Singh, Atticus Geiger, Bertie Vidgen, Christopher Potts, Divyansh Kaushik, Douwe Kiela, Grusha Prasad, Max Bartolo, Mohit Bansal, Pontus Stenetorp, Pratik Ringshia, Robin Jia, Sebastian Riedel, Tristan Thrush, Yixin Nie, Zeerak Waseem, Zhengxuan Wu, Zhiyi Ma","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-07T17:49:17Z","title":"Dynabench: Rethinking Benchmarking in NLP"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.14337","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:695a9463b81644ca5f8cf0f7b8351915718e91373a568e7d08c636a6f754fdef","target":"record","created_at":"2026-07-05T02:36:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"91e1e43c0e7aac90b20ca2651ee973a889c85830dd1dc64403a703c0a08d561b","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-07T17:49:17Z","title_canon_sha256":"a60eebbdeedb78e3a6421b8e70cdd26279b1dfe4e7c8b185b7095c20d541680a"},"schema_version":"1.0","source":{"id":"2104.14337","kind":"arxiv","version":1}},"canonical_sha256":"ea408bef2c5a85baf9ee323c1abc09d8d989db6c26e13140388a869824caba15","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ea408bef2c5a85baf9ee323c1abc09d8d989db6c26e13140388a869824caba15","first_computed_at":"2026-07-05T02:36:15.475559Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:36:15.475559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Jjfhv6cY6Qr/0oQD3VgJ+GBaoT85OB2bMk1UIosz3ALvyadWS6rk3K3xJSOsu0D1x6uWLx42mf4av1RGKsjeAw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:36:15.475983Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.14337","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:695a9463b81644ca5f8cf0f7b8351915718e91373a568e7d08c636a6f754fdef","sha256:84833472e757e990bd8d0485ca30b02d3e4e284c2cef7bf73bf7d74caef09910"],"state_sha256":"8d72c3ca3ea046a9e35d3bbfd31bfed6b82f2373a2339b9b51cc5c63b074f905"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ayTxRWsD4AyGnzUteYPJN7q3OIvlVY6yQmkEJl44fqaemzZirassOL3kY6aioAYEb9uD04H5vLu/GzqHg/IRCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T17:21:30.644271Z","bundle_sha256":"b311e15bcc0a781ca5df700945f7520105e041272ba25e72639db128bd6c303f"}}