{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:LPNSYHKKH5HF6DJJTOCPXQ54ES","short_pith_number":"pith:LPNSYHKK","schema_version":"1.0","canonical_sha256":"5bdb2c1d4a3f4e5f0d299b84fbc3bc249944486f1fd328260af9a1543c4e4777","source":{"kind":"arxiv","id":"2403.06644","version":1},"attestation_state":"computed","paper":{"title":"Elephants Never Forget: Testing Language Models for Memorization of Tabular Data","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Harsha Nori, Rich Caruana, Sebastian Bordt","submitted_at":"2024-03-11T12:07:13Z","abstract_excerpt":"While many have shown how Large Language Models (LLMs) can be applied to a diverse set of tasks, the critical issues of data contamination and memorization are often glossed over. In this work, we address this concern for tabular data. Starting with simple qualitative tests for whether an LLM knows the names and values of features, we introduce a variety of different techniques to assess the degrees of contamination, including statistical tests for conditional distribution modeling and four tests that identify memorization. Our investigation reveals that LLMs are pre-trained on many popular ta"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2403.06644","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-11T12:07:13Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"6b9085f45403cb2fbb87fa32f5047c51601a8665ae27ee52d533dac687c908e2","abstract_canon_sha256":"bfbe7883ac86d4db40ee3fdb6be25f7079aed9cb76047af4ccfcb8980a213c69"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:54:40.748024Z","signature_b64":"bYCxffsabm/AMO2oNK4ArDWernyyJpCvDxh2VzQ9eJAaEq2YXiHgPdDyIf3rxZeQc7FBqoEyoSucQaFpqEhXDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5bdb2c1d4a3f4e5f0d299b84fbc3bc249944486f1fd328260af9a1543c4e4777","last_reissued_at":"2026-07-05T07:54:40.747524Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:54:40.747524Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Elephants Never Forget: Testing Language Models for Memorization of Tabular Data","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Harsha Nori, Rich Caruana, Sebastian Bordt","submitted_at":"2024-03-11T12:07:13Z","abstract_excerpt":"While many have shown how Large Language Models (LLMs) can be applied to a diverse set of tasks, the critical issues of data contamination and memorization are often glossed over. In this work, we address this concern for tabular data. Starting with simple qualitative tests for whether an LLM knows the names and values of features, we introduce a variety of different techniques to assess the degrees of contamination, including statistical tests for conditional distribution modeling and four tests that identify memorization. Our investigation reveals that LLMs are pre-trained on many popular ta"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.06644","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.06644/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2403.06644","created_at":"2026-07-05T07:54:40.747584+00:00"},{"alias_kind":"arxiv_version","alias_value":"2403.06644v1","created_at":"2026-07-05T07:54:40.747584+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.06644","created_at":"2026-07-05T07:54:40.747584+00:00"},{"alias_kind":"pith_short_12","alias_value":"LPNSYHKKH5HF","created_at":"2026-07-05T07:54:40.747584+00:00"},{"alias_kind":"pith_short_16","alias_value":"LPNSYHKKH5HF6DJJ","created_at":"2026-07-05T07:54:40.747584+00:00"},{"alias_kind":"pith_short_8","alias_value":"LPNSYHKK","created_at":"2026-07-05T07:54:40.747584+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.31208","citing_title":"Probing Memorization of Tabular In-Context Learning","ref_index":184,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES","json":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES.json","graph_json":"https://pith.science/api/pith-number/LPNSYHKKH5HF6DJJTOCPXQ54ES/graph.json","events_json":"https://pith.science/api/pith-number/LPNSYHKKH5HF6DJJTOCPXQ54ES/events.json","paper":"https://pith.science/paper/LPNSYHKK"},"agent_actions":{"view_html":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES","download_json":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES.json","view_paper":"https://pith.science/paper/LPNSYHKK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2403.06644&json=true","fetch_graph":"https://pith.science/api/pith-number/LPNSYHKKH5HF6DJJTOCPXQ54ES/graph.json","fetch_events":"https://pith.science/api/pith-number/LPNSYHKKH5HF6DJJTOCPXQ54ES/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES/action/storage_attestation","attest_author":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES/action/author_attestation","sign_citation":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES/action/citation_signature","submit_replication":"https://pith.science/pith/LPNSYHKKH5HF6DJJTOCPXQ54ES/action/replication_record"}},"created_at":"2026-07-05T07:54:40.747584+00:00","updated_at":"2026-07-05T07:54:40.747584+00:00"}