{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:F3C5BRPGMIAPSXQVHXBRG2DJIX","short_pith_number":"pith:F3C5BRPG","canonical_record":{"source":{"id":"1911.00225","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-01T06:48:07Z","cross_cats_sorted":[],"title_canon_sha256":"78da689ee635bfcb509cdaae49184e037366ca194bbe052e6a60b4b7518d7a4e","abstract_canon_sha256":"c15c4edfb63d593eb31308be83d3489bcc30f32daa8efa2082d7b193b332fa76"},"schema_version":"1.0"},"canonical_sha256":"2ec5d0c5e66200f95e153dc313686945c9da4a3da7c427897cbb2637d9c11aae","source":{"kind":"arxiv","id":"1911.00225","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.00225","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"arxiv_version","alias_value":"1911.00225v1","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.00225","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_12","alias_value":"F3C5BRPGMIAP","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_16","alias_value":"F3C5BRPGMIAPSXQV","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_8","alias_value":"F3C5BRPG","created_at":"2026-07-05T00:16:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:F3C5BRPGMIAPSXQVHXBRG2DJIX","target":"record","payload":{"canonical_record":{"source":{"id":"1911.00225","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-01T06:48:07Z","cross_cats_sorted":[],"title_canon_sha256":"78da689ee635bfcb509cdaae49184e037366ca194bbe052e6a60b4b7518d7a4e","abstract_canon_sha256":"c15c4edfb63d593eb31308be83d3489bcc30f32daa8efa2082d7b193b332fa76"},"schema_version":"1.0"},"canonical_sha256":"2ec5d0c5e66200f95e153dc313686945c9da4a3da7c427897cbb2637d9c11aae","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:16:24.851343Z","signature_b64":"rVJ8WA0PCdtFOI+I/UqwzVKlSYtMMo/kFvfgyMbLiehYg5EDb0GgbrG/MBGXzK95O1ZSuvMYpOD2AhPv7yvUDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2ec5d0c5e66200f95e153dc313686945c9da4a3da7c427897cbb2637d9c11aae","last_reissued_at":"2026-07-05T00:16:24.850917Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:16:24.850917Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1911.00225","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:16:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"omJCs6PCfzTnPzCQ6GSMdBijisZfIQ6CPRbNcTzZUUO311/NILLXFQUG1KakahwCz6NT91E59FvsoFhmENRfDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T18:34:38.632277Z"},"content_sha256":"08f122414284910bdc46d12d5d3da99729d7d72ebd40c208f877260bcae38000","schema_version":"1.0","event_id":"sha256:08f122414284910bdc46d12d5d3da99729d7d72ebd40c208f877260bcae38000"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:F3C5BRPGMIAPSXQVHXBRG2DJIX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"When Choosing Plausible Alternatives, Clever Hans can be Clever","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Benjamin Heinzerling, Kentaro Inui, Keshav Singh, Naoya Inoue, Paul Reisert, Pride Kavumba","submitted_at":"2019-11-01T06:48:07Z","abstract_excerpt":"Pretrained language models, such as BERT and RoBERTa, have shown large improvements in the commonsense reasoning benchmark COPA. However, recent work found that many improvements in benchmarks of natural language understanding are not due to models learning the task, but due to their increasing ability to exploit superficial cues, such as tokens that occur more often in the correct answer than the wrong one. Are BERT's and RoBERTa's good performance on COPA also caused by this? We find superficial cues in COPA, as well as evidence that BERT exploits these cues. To remedy this problem, we intro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.00225","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1911.00225/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:16:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LT1oMoVECA19DuvE7tzsfJbAVOrl4EVq/Vz54NfcVQCl9Z+wUVcf9dI998wDik07onvi3jD2E4t7+oqNZZ3DDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T18:34:38.632793Z"},"content_sha256":"bcdbd6cc95934ac820fb14d6e7854390bad5cc69f13e16730673b15361be7915","schema_version":"1.0","event_id":"sha256:bcdbd6cc95934ac820fb14d6e7854390bad5cc69f13e16730673b15361be7915"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/bundle.json","state_url":"https://pith.science/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T18:34:38Z","links":{"resolver":"https://pith.science/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX","bundle":"https://pith.science/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/bundle.json","state":"https://pith.science/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F3C5BRPGMIAPSXQVHXBRG2DJIX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:F3C5BRPGMIAPSXQVHXBRG2DJIX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c15c4edfb63d593eb31308be83d3489bcc30f32daa8efa2082d7b193b332fa76","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-01T06:48:07Z","title_canon_sha256":"78da689ee635bfcb509cdaae49184e037366ca194bbe052e6a60b4b7518d7a4e"},"schema_version":"1.0","source":{"id":"1911.00225","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.00225","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"arxiv_version","alias_value":"1911.00225v1","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.00225","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_12","alias_value":"F3C5BRPGMIAP","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_16","alias_value":"F3C5BRPGMIAPSXQV","created_at":"2026-07-05T00:16:24Z"},{"alias_kind":"pith_short_8","alias_value":"F3C5BRPG","created_at":"2026-07-05T00:16:24Z"}],"graph_snapshots":[{"event_id":"sha256:bcdbd6cc95934ac820fb14d6e7854390bad5cc69f13e16730673b15361be7915","target":"graph","created_at":"2026-07-05T00:16:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1911.00225/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Pretrained language models, such as BERT and RoBERTa, have shown large improvements in the commonsense reasoning benchmark COPA. However, recent work found that many improvements in benchmarks of natural language understanding are not due to models learning the task, but due to their increasing ability to exploit superficial cues, such as tokens that occur more often in the correct answer than the wrong one. Are BERT's and RoBERTa's good performance on COPA also caused by this? We find superficial cues in COPA, as well as evidence that BERT exploits these cues. To remedy this problem, we intro","authors_text":"Benjamin Heinzerling, Kentaro Inui, Keshav Singh, Naoya Inoue, Paul Reisert, Pride Kavumba","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-01T06:48:07Z","title":"When Choosing Plausible Alternatives, Clever Hans can be Clever"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.00225","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:08f122414284910bdc46d12d5d3da99729d7d72ebd40c208f877260bcae38000","target":"record","created_at":"2026-07-05T00:16:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c15c4edfb63d593eb31308be83d3489bcc30f32daa8efa2082d7b193b332fa76","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-01T06:48:07Z","title_canon_sha256":"78da689ee635bfcb509cdaae49184e037366ca194bbe052e6a60b4b7518d7a4e"},"schema_version":"1.0","source":{"id":"1911.00225","kind":"arxiv","version":1}},"canonical_sha256":"2ec5d0c5e66200f95e153dc313686945c9da4a3da7c427897cbb2637d9c11aae","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2ec5d0c5e66200f95e153dc313686945c9da4a3da7c427897cbb2637d9c11aae","first_computed_at":"2026-07-05T00:16:24.850917Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:16:24.850917Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"rVJ8WA0PCdtFOI+I/UqwzVKlSYtMMo/kFvfgyMbLiehYg5EDb0GgbrG/MBGXzK95O1ZSuvMYpOD2AhPv7yvUDg==","signature_status":"signed_v1","signed_at":"2026-07-05T00:16:24.851343Z","signed_message":"canonical_sha256_bytes"},"source_id":"1911.00225","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:08f122414284910bdc46d12d5d3da99729d7d72ebd40c208f877260bcae38000","sha256:bcdbd6cc95934ac820fb14d6e7854390bad5cc69f13e16730673b15361be7915"],"state_sha256":"759662238b081cb1b1edda57067874ae9d1dcab1749c6ac70125426d91a15d26"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qQstFMuHK9/rXpRdmegE1hINoglf4xPtDgg0Qk5Tau/1fUOxh0qBgYFtAtZVhaPbxjmulQT0sxv2+f35Bv8sDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T18:34:38.638422Z","bundle_sha256":"0d65cb6fbdc4ac9b4dd0382cf594849d067b38cbfbd2fef437e952dc832be0fa"}}