{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:L5CFZK64AFNNBZQQIVA5DKPBD6","short_pith_number":"pith:L5CFZK64","canonical_record":{"source":{"id":"1911.02969","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-07T16:20:40Z","cross_cats_sorted":[],"title_canon_sha256":"64c15e0a760963ae6da59a81c112bab77c36ed9e54c8c32e4e76e18ddd8f6d68","abstract_canon_sha256":"4b5f72240b4e4bbb1b4ab69cfc4dff9836bb38e8c751b852634f6358c903b97a"},"schema_version":"1.0"},"canonical_sha256":"5f445cabdc015ad0e6104541d1a9e11fa7051f45b59a2caf34e7232a88b39c85","source":{"kind":"arxiv","id":"1911.02969","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.02969","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"arxiv_version","alias_value":"1911.02969v2","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.02969","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_12","alias_value":"L5CFZK64AFNN","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_16","alias_value":"L5CFZK64AFNNBZQQ","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_8","alias_value":"L5CFZK64","created_at":"2026-07-05T01:51:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:L5CFZK64AFNNBZQQIVA5DKPBD6","target":"record","payload":{"canonical_record":{"source":{"id":"1911.02969","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-07T16:20:40Z","cross_cats_sorted":[],"title_canon_sha256":"64c15e0a760963ae6da59a81c112bab77c36ed9e54c8c32e4e76e18ddd8f6d68","abstract_canon_sha256":"4b5f72240b4e4bbb1b4ab69cfc4dff9836bb38e8c751b852634f6358c903b97a"},"schema_version":"1.0"},"canonical_sha256":"5f445cabdc015ad0e6104541d1a9e11fa7051f45b59a2caf34e7232a88b39c85","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:51:37.711230Z","signature_b64":"BNMWUeikuCSw/CvI2S0Gr7I1Yy6cfdJFR8Ibpgtcu6/BnOw8XDQT5cQErwmgt7ykRg8uaxZdH4JN2RRjULquBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5f445cabdc015ad0e6104541d1a9e11fa7051f45b59a2caf34e7232a88b39c85","last_reissued_at":"2026-07-05T01:51:37.710854Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:51:37.710854Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1911.02969","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:51:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B/CsczV2kJNQNiPxWMwh4NHWY9imjkWYLl3LySY+xF8kSzouzV+NxhIFQbM4BvmU7lPQWqZs3C/EgltzDnEWDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:39:52.722337Z"},"content_sha256":"0961ee01de0e77ed3f310f76f5a70b9aafbd55993c73de6ff496b12d443d490e","schema_version":"1.0","event_id":"sha256:0961ee01de0e77ed3f310f76f5a70b9aafbd55993c73de6ff496b12d443d490e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:L5CFZK64AFNNBZQQIVA5DKPBD6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"BERTs of a feather do not generalize together: Large variability in generalization across models with similar test set performance","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Junghyun Min, R. Thomas McCoy, Tal Linzen","submitted_at":"2019-11-07T16:20:40Z","abstract_excerpt":"If the same neural network architecture is trained multiple times on the same dataset, will it make similar linguistic generalizations across runs? To study this question, we fine-tuned 100 instances of BERT on the Multi-genre Natural Language Inference (MNLI) dataset and evaluated them on the HANS dataset, which evaluates syntactic generalization in natural language inference. On the MNLI development set, the behavior of all instances was remarkably consistent, with accuracy ranging between 83.6% and 84.8%. In stark contrast, the same models varied widely in their generalization performance. "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.02969","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1911.02969/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:51:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2mKJCfYc2Ydz9scOezuOFct3PjoRihPUftHXM+zGYvzQpfdwqFcFwo7+U+v7PwQiw8psvOxml3rd8JM5HW6iCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T09:39:52.722843Z"},"content_sha256":"a2ca592ad9b73c6b874b0270cd40cf95f60e01a7cc4e74cc81b608402fc1a46c","schema_version":"1.0","event_id":"sha256:a2ca592ad9b73c6b874b0270cd40cf95f60e01a7cc4e74cc81b608402fc1a46c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/bundle.json","state_url":"https://pith.science/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T09:39:52Z","links":{"resolver":"https://pith.science/pith/L5CFZK64AFNNBZQQIVA5DKPBD6","bundle":"https://pith.science/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/bundle.json","state":"https://pith.science/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/L5CFZK64AFNNBZQQIVA5DKPBD6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:L5CFZK64AFNNBZQQIVA5DKPBD6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4b5f72240b4e4bbb1b4ab69cfc4dff9836bb38e8c751b852634f6358c903b97a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-07T16:20:40Z","title_canon_sha256":"64c15e0a760963ae6da59a81c112bab77c36ed9e54c8c32e4e76e18ddd8f6d68"},"schema_version":"1.0","source":{"id":"1911.02969","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1911.02969","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"arxiv_version","alias_value":"1911.02969v2","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1911.02969","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_12","alias_value":"L5CFZK64AFNN","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_16","alias_value":"L5CFZK64AFNNBZQQ","created_at":"2026-07-05T01:51:37Z"},{"alias_kind":"pith_short_8","alias_value":"L5CFZK64","created_at":"2026-07-05T01:51:37Z"}],"graph_snapshots":[{"event_id":"sha256:a2ca592ad9b73c6b874b0270cd40cf95f60e01a7cc4e74cc81b608402fc1a46c","target":"graph","created_at":"2026-07-05T01:51:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1911.02969/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"If the same neural network architecture is trained multiple times on the same dataset, will it make similar linguistic generalizations across runs? To study this question, we fine-tuned 100 instances of BERT on the Multi-genre Natural Language Inference (MNLI) dataset and evaluated them on the HANS dataset, which evaluates syntactic generalization in natural language inference. On the MNLI development set, the behavior of all instances was remarkably consistent, with accuracy ranging between 83.6% and 84.8%. In stark contrast, the same models varied widely in their generalization performance. ","authors_text":"Junghyun Min, R. Thomas McCoy, Tal Linzen","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-07T16:20:40Z","title":"BERTs of a feather do not generalize together: Large variability in generalization across models with similar test set performance"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1911.02969","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0961ee01de0e77ed3f310f76f5a70b9aafbd55993c73de6ff496b12d443d490e","target":"record","created_at":"2026-07-05T01:51:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4b5f72240b4e4bbb1b4ab69cfc4dff9836bb38e8c751b852634f6358c903b97a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-11-07T16:20:40Z","title_canon_sha256":"64c15e0a760963ae6da59a81c112bab77c36ed9e54c8c32e4e76e18ddd8f6d68"},"schema_version":"1.0","source":{"id":"1911.02969","kind":"arxiv","version":2}},"canonical_sha256":"5f445cabdc015ad0e6104541d1a9e11fa7051f45b59a2caf34e7232a88b39c85","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5f445cabdc015ad0e6104541d1a9e11fa7051f45b59a2caf34e7232a88b39c85","first_computed_at":"2026-07-05T01:51:37.710854Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:51:37.710854Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BNMWUeikuCSw/CvI2S0Gr7I1Yy6cfdJFR8Ibpgtcu6/BnOw8XDQT5cQErwmgt7ykRg8uaxZdH4JN2RRjULquBw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:51:37.711230Z","signed_message":"canonical_sha256_bytes"},"source_id":"1911.02969","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0961ee01de0e77ed3f310f76f5a70b9aafbd55993c73de6ff496b12d443d490e","sha256:a2ca592ad9b73c6b874b0270cd40cf95f60e01a7cc4e74cc81b608402fc1a46c"],"state_sha256":"7209b53f48a1f56c489af124b78c6f114ebea74f7f5b519648fa5fb6b38971d4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7f7HV5Q539KduvO9lyYA9lGfbtECQaY+dlT2dpwoFHgHZ+eNoXoNWjysCBxUkCNsOz7RloV3CChRvQsJtU7qBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T09:39:52.726146Z","bundle_sha256":"6be829c5036281d19c8e3d2ea34096a6f19deb74fe63596e5ce7162ec5182816"}}