{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:SFIMRYWTNZPBZVOUL5NWVUB5KQ","short_pith_number":"pith:SFIMRYWT","canonical_record":{"source":{"id":"2106.00840","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-01T22:33:53Z","cross_cats_sorted":[],"title_canon_sha256":"ae7eda9962558c6a9529567a65602b7ad2fb8aaf4e312e9fa08514f21c1a8e45","abstract_canon_sha256":"bbc214e776b52008500dd1907f1cd4ffbc9c39f5ef32b78fe3275558a1fe7375"},"schema_version":"1.0"},"canonical_sha256":"9150c8e2d36e5e1cd5d45f5b6ad03d541e2c15388ae72610a15a1a92d0d2d81b","source":{"kind":"arxiv","id":"2106.00840","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.00840","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"arxiv_version","alias_value":"2106.00840v1","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.00840","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_12","alias_value":"SFIMRYWTNZPB","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_16","alias_value":"SFIMRYWTNZPBZVOU","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_8","alias_value":"SFIMRYWT","created_at":"2026-07-05T02:45:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:SFIMRYWTNZPBZVOUL5NWVUB5KQ","target":"record","payload":{"canonical_record":{"source":{"id":"2106.00840","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-01T22:33:53Z","cross_cats_sorted":[],"title_canon_sha256":"ae7eda9962558c6a9529567a65602b7ad2fb8aaf4e312e9fa08514f21c1a8e45","abstract_canon_sha256":"bbc214e776b52008500dd1907f1cd4ffbc9c39f5ef32b78fe3275558a1fe7375"},"schema_version":"1.0"},"canonical_sha256":"9150c8e2d36e5e1cd5d45f5b6ad03d541e2c15388ae72610a15a1a92d0d2d81b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:45:40.531019Z","signature_b64":"WL0VFNAD/nWlfkZavJElRDreQcWamhjCtAht1yBUwZcu9X1gEGEWXHA9TYSPMfqNEyFOjIBwkp4gTa2ulLvgCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9150c8e2d36e5e1cd5d45f5b6ad03d541e2c15388ae72610a15a1a92d0d2d81b","last_reissued_at":"2026-07-05T02:45:40.530555Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:45:40.530555Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2106.00840","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:45:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0XG1aBLBLeRyzOhO94HPlQ5f6/xdXucCMJeuXwOSjKXuDOgSVum0XMC6EDej+GnC8H4hGJGM6Vm0kKPaU0NgAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:37:41.505616Z"},"content_sha256":"69ce8eba9e0fc3a710d69bbefc126725d3ad64d4a2a4143893dd16f17e6ba91b","schema_version":"1.0","event_id":"sha256:69ce8eba9e0fc3a710d69bbefc126725d3ad64d4a2a4143893dd16f17e6ba91b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:SFIMRYWTNZPBZVOUL5NWVUB5KQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Comparing Test Sets with Item Response Theory","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Clara Vania, Dhara Mungra, Haokun Liu, Jason Phang, Kyunghyun Cho, Phu Mon Htut, Richard Yuanzhe Pang, Samuel R. Bowman, William Huang","submitted_at":"2021-06-01T22:33:53Z","abstract_excerpt":"Recent years have seen numerous NLP datasets introduced to evaluate the performance of fine-tuned models on natural language understanding tasks. Recent results from large pretrained models, though, show that many of these datasets are largely saturated and unlikely to be able to detect further progress. What kind of datasets are still effective at discriminating among strong models, and what kind of datasets should we expect to be able to detect future improvements? To measure this uniformly across datasets, we draw on Item Response Theory and evaluate 29 datasets using predictions from 18 pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.00840","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.00840/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:45:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nFNweWSwJsnOXpGbF77461wkxRpyQcvwMWO+3prmeoSS3k3Cy9R6tDgr3ZGxm2gydV8nIni05VXVx+LK7W0CCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:37:41.506466Z"},"content_sha256":"45ec9bc8298d7a8c4b6d834bfcedfbfbce7414dfca957869efdd17c44ea51837","schema_version":"1.0","event_id":"sha256:45ec9bc8298d7a8c4b6d834bfcedfbfbce7414dfca957869efdd17c44ea51837"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/bundle.json","state_url":"https://pith.science/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T19:37:41Z","links":{"resolver":"https://pith.science/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ","bundle":"https://pith.science/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/bundle.json","state":"https://pith.science/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/SFIMRYWTNZPBZVOUL5NWVUB5KQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:SFIMRYWTNZPBZVOUL5NWVUB5KQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bbc214e776b52008500dd1907f1cd4ffbc9c39f5ef32b78fe3275558a1fe7375","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-01T22:33:53Z","title_canon_sha256":"ae7eda9962558c6a9529567a65602b7ad2fb8aaf4e312e9fa08514f21c1a8e45"},"schema_version":"1.0","source":{"id":"2106.00840","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.00840","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"arxiv_version","alias_value":"2106.00840v1","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.00840","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_12","alias_value":"SFIMRYWTNZPB","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_16","alias_value":"SFIMRYWTNZPBZVOU","created_at":"2026-07-05T02:45:40Z"},{"alias_kind":"pith_short_8","alias_value":"SFIMRYWT","created_at":"2026-07-05T02:45:40Z"}],"graph_snapshots":[{"event_id":"sha256:45ec9bc8298d7a8c4b6d834bfcedfbfbce7414dfca957869efdd17c44ea51837","target":"graph","created_at":"2026-07-05T02:45:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2106.00840/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent years have seen numerous NLP datasets introduced to evaluate the performance of fine-tuned models on natural language understanding tasks. Recent results from large pretrained models, though, show that many of these datasets are largely saturated and unlikely to be able to detect further progress. What kind of datasets are still effective at discriminating among strong models, and what kind of datasets should we expect to be able to detect future improvements? To measure this uniformly across datasets, we draw on Item Response Theory and evaluate 29 datasets using predictions from 18 pr","authors_text":"Clara Vania, Dhara Mungra, Haokun Liu, Jason Phang, Kyunghyun Cho, Phu Mon Htut, Richard Yuanzhe Pang, Samuel R. Bowman, William Huang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-01T22:33:53Z","title":"Comparing Test Sets with Item Response Theory"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.00840","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:69ce8eba9e0fc3a710d69bbefc126725d3ad64d4a2a4143893dd16f17e6ba91b","target":"record","created_at":"2026-07-05T02:45:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bbc214e776b52008500dd1907f1cd4ffbc9c39f5ef32b78fe3275558a1fe7375","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-01T22:33:53Z","title_canon_sha256":"ae7eda9962558c6a9529567a65602b7ad2fb8aaf4e312e9fa08514f21c1a8e45"},"schema_version":"1.0","source":{"id":"2106.00840","kind":"arxiv","version":1}},"canonical_sha256":"9150c8e2d36e5e1cd5d45f5b6ad03d541e2c15388ae72610a15a1a92d0d2d81b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9150c8e2d36e5e1cd5d45f5b6ad03d541e2c15388ae72610a15a1a92d0d2d81b","first_computed_at":"2026-07-05T02:45:40.530555Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:45:40.530555Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WL0VFNAD/nWlfkZavJElRDreQcWamhjCtAht1yBUwZcu9X1gEGEWXHA9TYSPMfqNEyFOjIBwkp4gTa2ulLvgCg==","signature_status":"signed_v1","signed_at":"2026-07-05T02:45:40.531019Z","signed_message":"canonical_sha256_bytes"},"source_id":"2106.00840","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:69ce8eba9e0fc3a710d69bbefc126725d3ad64d4a2a4143893dd16f17e6ba91b","sha256:45ec9bc8298d7a8c4b6d834bfcedfbfbce7414dfca957869efdd17c44ea51837"],"state_sha256":"9030027f55d8ceefa49cf17c8c832b6a86dacf1e3eb52c838bf288c1a62d77a3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iIOJNUOjFUdXQyVx7Jq7eS0ugohz7vNwHZ0gC7YWn3UyIH1/kbpc/ItdHJ+mpHSXBS4SdCnsQT0p6RG8xUOnDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T19:37:41.511597Z","bundle_sha256":"ac62b4abb33dbd0e8c8139f68d948cd67c69a744b3654c9a5679ebd0302db11a"}}