{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:P4436IWO2SKOPSE3VV25WXQTRN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e40eb5e9c1a76136d23788079b10231691fcad818d278d7af549fba59f38da85","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-28T08:28:09Z","title_canon_sha256":"648f66d7ce4b6be5491a3e324c7677d0e697a19df5a6584199e9f28e015a080e"},"schema_version":"1.0","source":{"id":"2207.13948","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2207.13948","created_at":"2026-07-05T04:44:22Z"},{"alias_kind":"arxiv_version","alias_value":"2207.13948v1","created_at":"2026-07-05T04:44:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2207.13948","created_at":"2026-07-05T04:44:22Z"},{"alias_kind":"pith_short_12","alias_value":"P4436IWO2SKO","created_at":"2026-07-05T04:44:22Z"},{"alias_kind":"pith_short_16","alias_value":"P4436IWO2SKOPSE3","created_at":"2026-07-05T04:44:22Z"},{"alias_kind":"pith_short_8","alias_value":"P4436IWO","created_at":"2026-07-05T04:44:22Z"}],"graph_snapshots":[{"event_id":"sha256:0ce679ff7c1a66ce2cb8c411df2a3327fbd1a44fdb844cf64bec29986eed5e14","target":"graph","created_at":"2026-07-05T04:44:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2207.13948/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While pre-trained language models (LMs) have brought great improvements in many NLP tasks, there is increasing attention to explore capabilities of LMs and interpret their predictions. However, existing works usually focus only on a certain capability with some downstream tasks. There is a lack of datasets for directly evaluating the masked word prediction performance and the interpretability of pre-trained LMs. To fill in the gap, we propose a novel evaluation benchmark providing with both English and Chinese annotated data. It tests LMs abilities in multiple dimensions, i.e., grammar, semant","authors_text":"Hua Wu, Jing Liu, Lijie Wang, Xinyan Xiao, Yaozong Shen, Ying Chen","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-28T08:28:09Z","title":"An Interpretability Evaluation Benchmark for Pre-trained Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2207.13948","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3e5c9443e96895cdf44cb186dae206b2d29ece22eee359845b9dcf0fe336157f","target":"record","created_at":"2026-07-05T04:44:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e40eb5e9c1a76136d23788079b10231691fcad818d278d7af549fba59f38da85","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-07-28T08:28:09Z","title_canon_sha256":"648f66d7ce4b6be5491a3e324c7677d0e697a19df5a6584199e9f28e015a080e"},"schema_version":"1.0","source":{"id":"2207.13948","kind":"arxiv","version":1}},"canonical_sha256":"7f39bf22ced494e7c89bad75db5e138b6a01ef11cecebae67c5a1ccf06fd4e5c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7f39bf22ced494e7c89bad75db5e138b6a01ef11cecebae67c5a1ccf06fd4e5c","first_computed_at":"2026-07-05T04:44:22.995878Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:44:22.995878Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9kRu0YinhL7trL2XsrrewNo0VCaSo0kOhoSFATCnEMi7tjh4uA0/ZeY5inwwRlcXOzJ8fTY5Va98Nx/39naaBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T04:44:22.996333Z","signed_message":"canonical_sha256_bytes"},"source_id":"2207.13948","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3e5c9443e96895cdf44cb186dae206b2d29ece22eee359845b9dcf0fe336157f","sha256:0ce679ff7c1a66ce2cb8c411df2a3327fbd1a44fdb844cf64bec29986eed5e14"],"state_sha256":"003ba42e878d8634d1e71f0d0675961522f78fa6a8661e881a3502c534e4cdfe"}