{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:UKGQYGKATVZRWNTVNRSBRO7RGV","short_pith_number":"pith:UKGQYGKA","canonical_record":{"source":{"id":"2004.15001","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-04-30T17:47:17Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"bef623f333c3ed39cbc972ef9b1d9a5e127bf559cf01029c8b327433ffa2ec67","abstract_canon_sha256":"0bb2ca211629d697d7ed40752563fd6485761d7f442dee21d03b2f3731518b9c"},"schema_version":"1.0"},"canonical_sha256":"a28d0c19409d731b36756c6418bbf13554fc9df2cf5d90a6617339f1dae0a56c","source":{"kind":"arxiv","id":"2004.15001","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.15001","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"arxiv_version","alias_value":"2004.15001v2","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.15001","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_12","alias_value":"UKGQYGKATVZR","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_16","alias_value":"UKGQYGKATVZRWNTV","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_8","alias_value":"UKGQYGKA","created_at":"2026-07-05T01:40:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:UKGQYGKATVZRWNTVNRSBRO7RGV","target":"record","payload":{"canonical_record":{"source":{"id":"2004.15001","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-04-30T17:47:17Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"bef623f333c3ed39cbc972ef9b1d9a5e127bf559cf01029c8b327433ffa2ec67","abstract_canon_sha256":"0bb2ca211629d697d7ed40752563fd6485761d7f442dee21d03b2f3731518b9c"},"schema_version":"1.0"},"canonical_sha256":"a28d0c19409d731b36756c6418bbf13554fc9df2cf5d90a6617339f1dae0a56c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:40:27.568837Z","signature_b64":"V2uyogfXlRaS6D5tOZ1gA7zhF9EgzbATfCLq49lyIqahqNzyNd5uWB5D8quQ3dpM8A2Taaz3/xGUDWVjYgJWAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a28d0c19409d731b36756c6418bbf13554fc9df2cf5d90a6617339f1dae0a56c","last_reissued_at":"2026-07-05T01:40:27.568433Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:40:27.568433Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2004.15001","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"g/AZkp1+v5/NUYdLxBC8kDO5NvITF5Gxc/7PShmxjmRpZv35IPKkv/dsjsNbwmDy3iC/8u3a3UIdMlf8wP+WBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T16:45:36.804326Z"},"content_sha256":"910f9ffeb3101848846e7f0128b11756574e290fb07923be9d1b0033cebaaf4a","schema_version":"1.0","event_id":"sha256:910f9ffeb3101848846e7f0128b11756574e290fb07923be9d1b0033cebaaf4a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:UKGQYGKATVZRWNTVNRSBRO7RGV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Don't Use English Dev: On the Zero-Shot Cross-Lingual Evaluation of Contextual Embeddings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Julian Salazar, Phillip Keung, Vikas Bhardwaj, Yichao Lu","submitted_at":"2020-04-30T17:47:17Z","abstract_excerpt":"Multilingual contextual embeddings have demonstrated state-of-the-art performance in zero-shot cross-lingual transfer learning, where multilingual BERT is fine-tuned on one source language and evaluated on a different target language. However, published results for mBERT zero-shot accuracy vary as much as 17 points on the MLDoc classification task across four papers. We show that the standard practice of using English dev accuracy for model selection in the zero-shot setting makes it difficult to obtain reproducible results on the MLDoc and XNLI tasks. English dev accuracy is often uncorrelate"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.15001","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.15001/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QxcOshUlR11JnQR8OarXb4Kw8teBwbVnKNTQTVjgusNPecuaaSZnuWFjSF7yGFsp2odWymB6iINCrlkjoz6CBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T16:45:36.804603Z"},"content_sha256":"93c0cd90c263ae00c1818705555020a7d38b2743298c9a6755d05a100dbaa717","schema_version":"1.0","event_id":"sha256:93c0cd90c263ae00c1818705555020a7d38b2743298c9a6755d05a100dbaa717"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/bundle.json","state_url":"https://pith.science/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T16:45:36Z","links":{"resolver":"https://pith.science/pith/UKGQYGKATVZRWNTVNRSBRO7RGV","bundle":"https://pith.science/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/bundle.json","state":"https://pith.science/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UKGQYGKATVZRWNTVNRSBRO7RGV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:UKGQYGKATVZRWNTVNRSBRO7RGV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0bb2ca211629d697d7ed40752563fd6485761d7f442dee21d03b2f3731518b9c","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-04-30T17:47:17Z","title_canon_sha256":"bef623f333c3ed39cbc972ef9b1d9a5e127bf559cf01029c8b327433ffa2ec67"},"schema_version":"1.0","source":{"id":"2004.15001","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.15001","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"arxiv_version","alias_value":"2004.15001v2","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.15001","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_12","alias_value":"UKGQYGKATVZR","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_16","alias_value":"UKGQYGKATVZRWNTV","created_at":"2026-07-05T01:40:27Z"},{"alias_kind":"pith_short_8","alias_value":"UKGQYGKA","created_at":"2026-07-05T01:40:27Z"}],"graph_snapshots":[{"event_id":"sha256:93c0cd90c263ae00c1818705555020a7d38b2743298c9a6755d05a100dbaa717","target":"graph","created_at":"2026-07-05T01:40:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.15001/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multilingual contextual embeddings have demonstrated state-of-the-art performance in zero-shot cross-lingual transfer learning, where multilingual BERT is fine-tuned on one source language and evaluated on a different target language. However, published results for mBERT zero-shot accuracy vary as much as 17 points on the MLDoc classification task across four papers. We show that the standard practice of using English dev accuracy for model selection in the zero-shot setting makes it difficult to obtain reproducible results on the MLDoc and XNLI tasks. English dev accuracy is often uncorrelate","authors_text":"Julian Salazar, Phillip Keung, Vikas Bhardwaj, Yichao Lu","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-04-30T17:47:17Z","title":"Don't Use English Dev: On the Zero-Shot Cross-Lingual Evaluation of Contextual Embeddings"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.15001","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:910f9ffeb3101848846e7f0128b11756574e290fb07923be9d1b0033cebaaf4a","target":"record","created_at":"2026-07-05T01:40:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0bb2ca211629d697d7ed40752563fd6485761d7f442dee21d03b2f3731518b9c","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-04-30T17:47:17Z","title_canon_sha256":"bef623f333c3ed39cbc972ef9b1d9a5e127bf559cf01029c8b327433ffa2ec67"},"schema_version":"1.0","source":{"id":"2004.15001","kind":"arxiv","version":2}},"canonical_sha256":"a28d0c19409d731b36756c6418bbf13554fc9df2cf5d90a6617339f1dae0a56c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a28d0c19409d731b36756c6418bbf13554fc9df2cf5d90a6617339f1dae0a56c","first_computed_at":"2026-07-05T01:40:27.568433Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:40:27.568433Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"V2uyogfXlRaS6D5tOZ1gA7zhF9EgzbATfCLq49lyIqahqNzyNd5uWB5D8quQ3dpM8A2Taaz3/xGUDWVjYgJWAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T01:40:27.568837Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.15001","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:910f9ffeb3101848846e7f0128b11756574e290fb07923be9d1b0033cebaaf4a","sha256:93c0cd90c263ae00c1818705555020a7d38b2743298c9a6755d05a100dbaa717"],"state_sha256":"40e8d4fd9060623beeeb7d735472ccc229930714c574afc8187bca9757137a67"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xp/kkV6uFhCLMJ/a0pxA9pkRum0RLK5mLqkBounH48TusKhS6Ci0NbNNlBz0mOeef4eKVoyd376h5K1sMEYjDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T16:45:36.807739Z","bundle_sha256":"ced5df7afd76ec3fad825b3297064ec5e66d5b17ceca48a982c319b044b96e99"}}