{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:72DQD4376FFCMWCQ4BOA57IFOH","short_pith_number":"pith:72DQD437","canonical_record":{"source":{"id":"2102.01818","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-02-03T00:58:45Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"7a915cbab84dd2e3ab94adc20c9d3a2cde8ecdc4edb5d265f470c7242205ecc5","abstract_canon_sha256":"caaea778cef776d92d566bf77273d0136b39135343975bc916c6b1885c7f8198"},"schema_version":"1.0"},"canonical_sha256":"fe8701f37ff14a265850e05c0efd0571dd436bafad69abede818449fc5ac379e","source":{"kind":"arxiv","id":"2102.01818","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.01818","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"arxiv_version","alias_value":"2102.01818v1","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.01818","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_12","alias_value":"72DQD4376FFC","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_16","alias_value":"72DQD4376FFCMWCQ","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_8","alias_value":"72DQD437","created_at":"2026-07-05T02:12:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:72DQD4376FFCMWCQ4BOA57IFOH","target":"record","payload":{"canonical_record":{"source":{"id":"2102.01818","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-02-03T00:58:45Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"7a915cbab84dd2e3ab94adc20c9d3a2cde8ecdc4edb5d265f470c7242205ecc5","abstract_canon_sha256":"caaea778cef776d92d566bf77273d0136b39135343975bc916c6b1885c7f8198"},"schema_version":"1.0"},"canonical_sha256":"fe8701f37ff14a265850e05c0efd0571dd436bafad69abede818449fc5ac379e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:12:33.785312Z","signature_b64":"VxX9T69leh9pABobc5woarCcnE2HdaJ0P+urIIynHMXTrrlf2I5rsuK5k9ZUYJrmIYWAu07rOQV1s8RTV/AkDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fe8701f37ff14a265850e05c0efd0571dd436bafad69abede818449fc5ac379e","last_reissued_at":"2026-07-05T02:12:33.784847Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:12:33.784847Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2102.01818","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:12:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+9K2PyEdBJnVhmI/6dLHdx5DRU85AFMHlHb0ErhFKUgxhQtyJwwuFAI2vGHyBcrvrOXjjez75m6CCUUFK0csCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T02:47:08.001289Z"},"content_sha256":"a915771f281df7c00fdd4773fd92410c4d84304791e46a8fb352b58c78adb734","schema_version":"1.0","event_id":"sha256:a915771f281df7c00fdd4773fd92410c4d84304791e46a8fb352b58c78adb734"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:72DQD4376FFCMWCQ4BOA57IFOH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Memorization vs. Generalization: Quantifying Data Leakage in NLP Performance Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Aparna Elangovan, Jiayuan He, Karin Verspoor","submitted_at":"2021-02-03T00:58:45Z","abstract_excerpt":"Public datasets are often used to evaluate the efficacy and generalizability of state-of-the-art methods for many tasks in natural language processing (NLP). However, the presence of overlap between the train and test datasets can lead to inflated results, inadvertently evaluating the model's ability to memorize and interpreting it as the ability to generalize. In addition, such data sets may not provide an effective indicator of the performance of these methods in real world scenarios. We identify leakage of training data into test data on several publicly available datasets used to evaluate "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.01818","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2102.01818/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:12:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6qtVrfW09d4fC0ei9dNmvvuDVF6iLjzKf5POA8POqcm+u2Ph5HQfngyrcjHjRvg/pe3oJvYbPNJKF2E1w0ocBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T02:47:08.001810Z"},"content_sha256":"76775bb584a162cabf7900667e8ea5326682fa6fb6ee29db6f9f2d96e02205d8","schema_version":"1.0","event_id":"sha256:76775bb584a162cabf7900667e8ea5326682fa6fb6ee29db6f9f2d96e02205d8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/72DQD4376FFCMWCQ4BOA57IFOH/bundle.json","state_url":"https://pith.science/pith/72DQD4376FFCMWCQ4BOA57IFOH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/72DQD4376FFCMWCQ4BOA57IFOH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T02:47:08Z","links":{"resolver":"https://pith.science/pith/72DQD4376FFCMWCQ4BOA57IFOH","bundle":"https://pith.science/pith/72DQD4376FFCMWCQ4BOA57IFOH/bundle.json","state":"https://pith.science/pith/72DQD4376FFCMWCQ4BOA57IFOH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/72DQD4376FFCMWCQ4BOA57IFOH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:72DQD4376FFCMWCQ4BOA57IFOH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"caaea778cef776d92d566bf77273d0136b39135343975bc916c6b1885c7f8198","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-02-03T00:58:45Z","title_canon_sha256":"7a915cbab84dd2e3ab94adc20c9d3a2cde8ecdc4edb5d265f470c7242205ecc5"},"schema_version":"1.0","source":{"id":"2102.01818","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.01818","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"arxiv_version","alias_value":"2102.01818v1","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.01818","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_12","alias_value":"72DQD4376FFC","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_16","alias_value":"72DQD4376FFCMWCQ","created_at":"2026-07-05T02:12:33Z"},{"alias_kind":"pith_short_8","alias_value":"72DQD437","created_at":"2026-07-05T02:12:33Z"}],"graph_snapshots":[{"event_id":"sha256:76775bb584a162cabf7900667e8ea5326682fa6fb6ee29db6f9f2d96e02205d8","target":"graph","created_at":"2026-07-05T02:12:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2102.01818/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Public datasets are often used to evaluate the efficacy and generalizability of state-of-the-art methods for many tasks in natural language processing (NLP). However, the presence of overlap between the train and test datasets can lead to inflated results, inadvertently evaluating the model's ability to memorize and interpreting it as the ability to generalize. In addition, such data sets may not provide an effective indicator of the performance of these methods in real world scenarios. We identify leakage of training data into test data on several publicly available datasets used to evaluate ","authors_text":"Aparna Elangovan, Jiayuan He, Karin Verspoor","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-02-03T00:58:45Z","title":"Memorization vs. Generalization: Quantifying Data Leakage in NLP Performance Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.01818","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a915771f281df7c00fdd4773fd92410c4d84304791e46a8fb352b58c78adb734","target":"record","created_at":"2026-07-05T02:12:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"caaea778cef776d92d566bf77273d0136b39135343975bc916c6b1885c7f8198","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-02-03T00:58:45Z","title_canon_sha256":"7a915cbab84dd2e3ab94adc20c9d3a2cde8ecdc4edb5d265f470c7242205ecc5"},"schema_version":"1.0","source":{"id":"2102.01818","kind":"arxiv","version":1}},"canonical_sha256":"fe8701f37ff14a265850e05c0efd0571dd436bafad69abede818449fc5ac379e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fe8701f37ff14a265850e05c0efd0571dd436bafad69abede818449fc5ac379e","first_computed_at":"2026-07-05T02:12:33.784847Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:12:33.784847Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"VxX9T69leh9pABobc5woarCcnE2HdaJ0P+urIIynHMXTrrlf2I5rsuK5k9ZUYJrmIYWAu07rOQV1s8RTV/AkDw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:12:33.785312Z","signed_message":"canonical_sha256_bytes"},"source_id":"2102.01818","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a915771f281df7c00fdd4773fd92410c4d84304791e46a8fb352b58c78adb734","sha256:76775bb584a162cabf7900667e8ea5326682fa6fb6ee29db6f9f2d96e02205d8"],"state_sha256":"0f10fb95c10d149cdb889df83eac41a2d2c225e39bc75a6fdc231dd1205829dc"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"m64rEaF0/zOjz7Fslo9tbQXvC6O8zrQUlUy/GmkuMVXTz5u3xRI00fosALTdxItuLxvd1YxkohwwcNggtqn6Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T02:47:08.004364Z","bundle_sha256":"98b46822c2478a4282c822a13265c1c67b0d1cfe152a8053f25fef1a4a927ba5"}}