{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:I7KLMSG442SAB5GFOH5EG7CIE2","short_pith_number":"pith:I7KLMSG4","canonical_record":{"source":{"id":"2205.11097","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T07:37:04Z","cross_cats_sorted":[],"title_canon_sha256":"20ead4a0d984fb8944234cb7edbf0122429f35f8d2986c3899ed0b906f51c1d3","abstract_canon_sha256":"440529be0c132b521bfe1e30028298f7d6f9f18e850c8c03d42aafb4203bc164"},"schema_version":"1.0"},"canonical_sha256":"47d4b648dce6a400f4c571fa437c4826a6ba324d1a935262174394b7289de006","source":{"kind":"arxiv","id":"2205.11097","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.11097","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"arxiv_version","alias_value":"2205.11097v2","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.11097","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_12","alias_value":"I7KLMSG442SA","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_16","alias_value":"I7KLMSG442SAB5GF","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_8","alias_value":"I7KLMSG4","created_at":"2026-07-05T05:16:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:I7KLMSG442SAB5GFOH5EG7CIE2","target":"record","payload":{"canonical_record":{"source":{"id":"2205.11097","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T07:37:04Z","cross_cats_sorted":[],"title_canon_sha256":"20ead4a0d984fb8944234cb7edbf0122429f35f8d2986c3899ed0b906f51c1d3","abstract_canon_sha256":"440529be0c132b521bfe1e30028298f7d6f9f18e850c8c03d42aafb4203bc164"},"schema_version":"1.0"},"canonical_sha256":"47d4b648dce6a400f4c571fa437c4826a6ba324d1a935262174394b7289de006","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:16:00.475087Z","signature_b64":"ioI9RJRwsPzDM2pPeqiyKiDIIIdaNVW4AnuifBNn9gJTpanZbYo3d7ScKfbSnLhujDdjDyUWtfdrFJuFu6xcAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"47d4b648dce6a400f4c571fa437c4826a6ba324d1a935262174394b7289de006","last_reissued_at":"2026-07-05T05:16:00.474625Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:16:00.474625Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2205.11097","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:16:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"99ORik/BDUp4fKviK9U83qJ6st+/HkfrdsXQDFrjesFCA9gIdYrHGKybOfk9urpUQzu2msdTJsqrN4JffsdgBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-19T09:33:36.245046Z"},"content_sha256":"acc4a5e3504277eff726250d69cc9206a785e3d8245e9f531dd6b2cae7f52255","schema_version":"1.0","event_id":"sha256:acc4a5e3504277eff726250d69cc9206a785e3d8245e9f531dd6b2cae7f52255"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:I7KLMSG442SAB5GFOH5EG7CIE2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Fine-grained Interpretability Evaluation Benchmark for Neural NLP","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"HaiFeng Wang, Hao Liu, Hongxuan Tang, Hua Wu, Lijie Wang, Shuai Zhang, Shuyuan Peng, Xinyan Xiao, Yaozong Shen, Ying Chen","submitted_at":"2022-05-23T07:37:04Z","abstract_excerpt":"While there is increasing concern about the interpretability of neural models, the evaluation of interpretability remains an open problem, due to the lack of proper evaluation datasets and metrics. In this paper, we present a novel benchmark to evaluate the interpretability of both neural models and saliency methods. This benchmark covers three representative NLP tasks: sentiment analysis, textual similarity and reading comprehension, each provided with both English and Chinese annotated data. In order to precisely evaluate the interpretability, we provide token-level rationales that are caref"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.11097","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2205.11097/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:16:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6qCcFHwkOYt+5rdh3U1HL3tDEmWc7npy+e1KG+SkiH5MYkoUxc7WxaOMB5xWkNEUJEO5zrMLdyloPYAMLpdqCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-19T09:33:36.245437Z"},"content_sha256":"2f605789c9f36dec66b22b07815be64068ab697edd77e48155ee870a781da998","schema_version":"1.0","event_id":"sha256:2f605789c9f36dec66b22b07815be64068ab697edd77e48155ee870a781da998"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I7KLMSG442SAB5GFOH5EG7CIE2/bundle.json","state_url":"https://pith.science/pith/I7KLMSG442SAB5GFOH5EG7CIE2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I7KLMSG442SAB5GFOH5EG7CIE2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-19T09:33:36Z","links":{"resolver":"https://pith.science/pith/I7KLMSG442SAB5GFOH5EG7CIE2","bundle":"https://pith.science/pith/I7KLMSG442SAB5GFOH5EG7CIE2/bundle.json","state":"https://pith.science/pith/I7KLMSG442SAB5GFOH5EG7CIE2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I7KLMSG442SAB5GFOH5EG7CIE2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:I7KLMSG442SAB5GFOH5EG7CIE2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"440529be0c132b521bfe1e30028298f7d6f9f18e850c8c03d42aafb4203bc164","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T07:37:04Z","title_canon_sha256":"20ead4a0d984fb8944234cb7edbf0122429f35f8d2986c3899ed0b906f51c1d3"},"schema_version":"1.0","source":{"id":"2205.11097","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2205.11097","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"arxiv_version","alias_value":"2205.11097v2","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.11097","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_12","alias_value":"I7KLMSG442SA","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_16","alias_value":"I7KLMSG442SAB5GF","created_at":"2026-07-05T05:16:00Z"},{"alias_kind":"pith_short_8","alias_value":"I7KLMSG4","created_at":"2026-07-05T05:16:00Z"}],"graph_snapshots":[{"event_id":"sha256:2f605789c9f36dec66b22b07815be64068ab697edd77e48155ee870a781da998","target":"graph","created_at":"2026-07-05T05:16:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2205.11097/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While there is increasing concern about the interpretability of neural models, the evaluation of interpretability remains an open problem, due to the lack of proper evaluation datasets and metrics. In this paper, we present a novel benchmark to evaluate the interpretability of both neural models and saliency methods. This benchmark covers three representative NLP tasks: sentiment analysis, textual similarity and reading comprehension, each provided with both English and Chinese annotated data. In order to precisely evaluate the interpretability, we provide token-level rationales that are caref","authors_text":"HaiFeng Wang, Hao Liu, Hongxuan Tang, Hua Wu, Lijie Wang, Shuai Zhang, Shuyuan Peng, Xinyan Xiao, Yaozong Shen, Ying Chen","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T07:37:04Z","title":"A Fine-grained Interpretability Evaluation Benchmark for Neural NLP"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.11097","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:acc4a5e3504277eff726250d69cc9206a785e3d8245e9f531dd6b2cae7f52255","target":"record","created_at":"2026-07-05T05:16:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"440529be0c132b521bfe1e30028298f7d6f9f18e850c8c03d42aafb4203bc164","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-05-23T07:37:04Z","title_canon_sha256":"20ead4a0d984fb8944234cb7edbf0122429f35f8d2986c3899ed0b906f51c1d3"},"schema_version":"1.0","source":{"id":"2205.11097","kind":"arxiv","version":2}},"canonical_sha256":"47d4b648dce6a400f4c571fa437c4826a6ba324d1a935262174394b7289de006","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"47d4b648dce6a400f4c571fa437c4826a6ba324d1a935262174394b7289de006","first_computed_at":"2026-07-05T05:16:00.474625Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:16:00.474625Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ioI9RJRwsPzDM2pPeqiyKiDIIIdaNVW4AnuifBNn9gJTpanZbYo3d7ScKfbSnLhujDdjDyUWtfdrFJuFu6xcAA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:16:00.475087Z","signed_message":"canonical_sha256_bytes"},"source_id":"2205.11097","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:acc4a5e3504277eff726250d69cc9206a785e3d8245e9f531dd6b2cae7f52255","sha256:2f605789c9f36dec66b22b07815be64068ab697edd77e48155ee870a781da998"],"state_sha256":"3e059370fa40a0a7dcefe3408d8dae997b41566b6adb21279acb13fe467d9b5b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dsA7s8Cza8xxq1Vc6ZyKjvhAko948gLGisVkGOb+rD7w2vqjuG3jmPZ5NzX7ndSG69cRCpD2MmTziUerLf9VBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-19T09:33:36.247582Z","bundle_sha256":"65e6efae3468af46d3a4a05df1805506dc56a8c033b8e854660c4a03de8d8160"}}