{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:ISEFKG7K66RHDXEROXNFOKNF4Y","short_pith_number":"pith:ISEFKG7K","canonical_record":{"source":{"id":"2110.08412","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-15T23:59:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5fd29a546e2f94b7ec95b122ff70de7a04a9593bbdcae8dc255a809f95db6fea","abstract_canon_sha256":"62c0923d43d67cbab49b1d7339ce3144267cc85dfc2cbce444623999f9e0841e"},"schema_version":"1.0"},"canonical_sha256":"4488551beaf7a271dc9175da5729a5e63d5ac4ca769b89f871b9876fb2a266e3","source":{"kind":"arxiv","id":"2110.08412","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.08412","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"arxiv_version","alias_value":"2110.08412v3","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.08412","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_12","alias_value":"ISEFKG7K66RH","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_16","alias_value":"ISEFKG7K66RHDXER","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_8","alias_value":"ISEFKG7K","created_at":"2026-07-05T05:12:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:ISEFKG7K66RHDXEROXNFOKNF4Y","target":"record","payload":{"canonical_record":{"source":{"id":"2110.08412","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-15T23:59:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"5fd29a546e2f94b7ec95b122ff70de7a04a9593bbdcae8dc255a809f95db6fea","abstract_canon_sha256":"62c0923d43d67cbab49b1d7339ce3144267cc85dfc2cbce444623999f9e0841e"},"schema_version":"1.0"},"canonical_sha256":"4488551beaf7a271dc9175da5729a5e63d5ac4ca769b89f871b9876fb2a266e3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:12:09.166308Z","signature_b64":"jnxeAdjm9BdYHRYv9jbHT8jtgykMW7r6yBjuKeBgJBrcbEAN9i4TYwEzUNrHM8H5tJKEU+r5zb6wwuOGgh7/DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4488551beaf7a271dc9175da5729a5e63d5ac4ca769b89f871b9876fb2a266e3","last_reissued_at":"2026-07-05T05:12:09.165809Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:12:09.165809Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2110.08412","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KdZP47nICN2MuABc+9YVF/nf3vDXf/8dAObyhcahONS6ZJAVi7QpPZJdZvHqfzcrOq4nW7t8R4Ct1EuhymBWCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T09:08:27.790803Z"},"content_sha256":"e2c8d606d80538edc7adb89e543c46896cb0b25b3e7b75895cc31a98a1e70507","schema_version":"1.0","event_id":"sha256:e2c8d606d80538edc7adb89e543c46896cb0b25b3e7b75895cc31a98a1e70507"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:ISEFKG7K66RHDXEROXNFOKNF4Y","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating the Faithfulness of Importance Measures in NLP by Recursively Masking Allegedly Important Tokens and Retraining","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Andreas Madsen, Nicholas Meade, Siva Reddy, Vaibhav Adlakha","submitted_at":"2021-10-15T23:59:42Z","abstract_excerpt":"To explain NLP models a popular approach is to use importance measures, such as attention, which inform input tokens are important for making a prediction. However, an open question is how well these explanations accurately reflect a model's logic, a property called faithfulness.\n  To answer this question, we propose Recursive ROAR, a new faithfulness metric. This works by recursively masking allegedly important tokens and then retraining the model. The principle is that this should result in worse model performance compared to masking random tokens. The result is a performance curve given a m"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.08412","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2110.08412/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"M4F6SOU+Gz2nzSFKEzBdwCHBIIcNphT6uoHSWpohf4m1IJtp8yKgqzvYBeBD6aXtPo//dcj2AG93hYDDyiOvDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T09:08:27.791180Z"},"content_sha256":"60f912f1dcfdeee6212fe1677b51b2f982b0a397014aba33cfe00c732d96020d","schema_version":"1.0","event_id":"sha256:60f912f1dcfdeee6212fe1677b51b2f982b0a397014aba33cfe00c732d96020d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/bundle.json","state_url":"https://pith.science/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-16T09:08:27Z","links":{"resolver":"https://pith.science/pith/ISEFKG7K66RHDXEROXNFOKNF4Y","bundle":"https://pith.science/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/bundle.json","state":"https://pith.science/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ISEFKG7K66RHDXEROXNFOKNF4Y/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:ISEFKG7K66RHDXEROXNFOKNF4Y","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"62c0923d43d67cbab49b1d7339ce3144267cc85dfc2cbce444623999f9e0841e","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-15T23:59:42Z","title_canon_sha256":"5fd29a546e2f94b7ec95b122ff70de7a04a9593bbdcae8dc255a809f95db6fea"},"schema_version":"1.0","source":{"id":"2110.08412","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.08412","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"arxiv_version","alias_value":"2110.08412v3","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.08412","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_12","alias_value":"ISEFKG7K66RH","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_16","alias_value":"ISEFKG7K66RHDXER","created_at":"2026-07-05T05:12:09Z"},{"alias_kind":"pith_short_8","alias_value":"ISEFKG7K","created_at":"2026-07-05T05:12:09Z"}],"graph_snapshots":[{"event_id":"sha256:60f912f1dcfdeee6212fe1677b51b2f982b0a397014aba33cfe00c732d96020d","target":"graph","created_at":"2026-07-05T05:12:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2110.08412/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"To explain NLP models a popular approach is to use importance measures, such as attention, which inform input tokens are important for making a prediction. However, an open question is how well these explanations accurately reflect a model's logic, a property called faithfulness.\n  To answer this question, we propose Recursive ROAR, a new faithfulness metric. This works by recursively masking allegedly important tokens and then retraining the model. The principle is that this should result in worse model performance compared to masking random tokens. The result is a performance curve given a m","authors_text":"Andreas Madsen, Nicholas Meade, Siva Reddy, Vaibhav Adlakha","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-15T23:59:42Z","title":"Evaluating the Faithfulness of Importance Measures in NLP by Recursively Masking Allegedly Important Tokens and Retraining"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.08412","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e2c8d606d80538edc7adb89e543c46896cb0b25b3e7b75895cc31a98a1e70507","target":"record","created_at":"2026-07-05T05:12:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"62c0923d43d67cbab49b1d7339ce3144267cc85dfc2cbce444623999f9e0841e","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-10-15T23:59:42Z","title_canon_sha256":"5fd29a546e2f94b7ec95b122ff70de7a04a9593bbdcae8dc255a809f95db6fea"},"schema_version":"1.0","source":{"id":"2110.08412","kind":"arxiv","version":3}},"canonical_sha256":"4488551beaf7a271dc9175da5729a5e63d5ac4ca769b89f871b9876fb2a266e3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4488551beaf7a271dc9175da5729a5e63d5ac4ca769b89f871b9876fb2a266e3","first_computed_at":"2026-07-05T05:12:09.165809Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:12:09.165809Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jnxeAdjm9BdYHRYv9jbHT8jtgykMW7r6yBjuKeBgJBrcbEAN9i4TYwEzUNrHM8H5tJKEU+r5zb6wwuOGgh7/DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T05:12:09.166308Z","signed_message":"canonical_sha256_bytes"},"source_id":"2110.08412","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e2c8d606d80538edc7adb89e543c46896cb0b25b3e7b75895cc31a98a1e70507","sha256:60f912f1dcfdeee6212fe1677b51b2f982b0a397014aba33cfe00c732d96020d"],"state_sha256":"2b13771a3eb118199eb751f1b73e525ea3d1c225d5c091c69cd042bae776ffaf"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"P00A9HYXoA17nDVg3D22ra/nnUFSX6IC9Xm/zpz6KgiXRr1aITcwSXH401/5oPzKnZt4nlg77uBKlSft9lk+Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-16T09:08:27.793715Z","bundle_sha256":"21f65bd772b1c76839bc820c01bd34ed2326c8bcaca1dbf069691049be87ec79"}}