{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:AVYX74M7QB6UASGVPFJM7RADFX","short_pith_number":"pith:AVYX74M7","canonical_record":{"source":{"id":"2310.16789","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-25T17:21:23Z","cross_cats_sorted":["cs.CR","cs.LG"],"title_canon_sha256":"eab3e6f8bd1ab37ec5727705f226b4a49b4c5d1b21a3097644eb670981474e65","abstract_canon_sha256":"c0edcfe54c48ba8be05af857532100278a4dad7c829824b0fd3ffbd5584273ec"},"schema_version":"1.0"},"canonical_sha256":"05717ff19f807d4048d57952cfc4032dfa072bf49732ab3f13cb7f60ae8cfb4a","source":{"kind":"arxiv","id":"2310.16789","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.16789","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"arxiv_version","alias_value":"2310.16789v3","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.16789","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"pith_short_12","alias_value":"AVYX74M7QB6U","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"AVYX74M7QB6UASGV","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"AVYX74M7","created_at":"2026-05-18T12:33:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:AVYX74M7QB6UASGVPFJM7RADFX","target":"record","payload":{"canonical_record":{"source":{"id":"2310.16789","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-25T17:21:23Z","cross_cats_sorted":["cs.CR","cs.LG"],"title_canon_sha256":"eab3e6f8bd1ab37ec5727705f226b4a49b4c5d1b21a3097644eb670981474e65","abstract_canon_sha256":"c0edcfe54c48ba8be05af857532100278a4dad7c829824b0fd3ffbd5584273ec"},"schema_version":"1.0"},"canonical_sha256":"05717ff19f807d4048d57952cfc4032dfa072bf49732ab3f13cb7f60ae8cfb4a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:13.447891Z","signature_b64":"YiGnw1vUnKo9G85xBdtxfQ/JiQg4os6tle9cTxsFYdf67aLj51O97Gj4IYd0U42om5vikFlJdpHf7kQi7S1qAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"05717ff19f807d4048d57952cfc4032dfa072bf49732ab3f13cb7f60ae8cfb4a","last_reissued_at":"2026-05-17T23:38:13.446899Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:13.446899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.16789","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Sg69U++q9GV0yNAr6A6MtxHClWwCPRDSBpWaUN1p3452POhNEsePNVkCdaIocvNkAgM/P3tTTC6uh0L0x09kBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-19T02:42:22.637434Z"},"content_sha256":"ac2317ad040080b6d887ab63cb9f95efb9c89c084bec08009290b57d4a5268d2","schema_version":"1.0","event_id":"sha256:ac2317ad040080b6d887ab63cb9f95efb9c89c084bec08009290b57d4a5268d2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:AVYX74M7QB6UASGVPFJM7RADFX","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Detecting Pretraining Data from Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens.","cross_cats":["cs.CR","cs.LG"],"primary_cat":"cs.CL","authors_text":"Anirudh Ajith, Danqi Chen, Daogao Liu, Luke Zettlemoyer, Mengzhou Xia, Terra Blevins, Weijia Shi, Yangsibo Huang","submitted_at":"2023-10-25T17:21:23Z","abstract_excerpt":"Although large language models (LLMs) are widely deployed, the data used to train them is rarely disclosed. Given the incredible scale of this data, up to trillions of tokens, it is all but certain that it includes potentially problematic text such as copyrighted materials, personally identifiable information, and test data for widely reported reference benchmarks. However, we currently have no way to know which data of these types is included or in what proportions. In this paper, we study the pretraining data detection problem: given a piece of text and black-box access to an LLM without kno"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Min-K% Prob achieves a 7.4% improvement on WIKIMIA over these previous methods. We apply Min-K% Prob to three real-world scenarios, copyrighted book detection, contaminated downstream example detection and privacy auditing of machine unlearning, and find it a consistently effective solution.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"An unseen example is likely to contain a few outlier words with low probabilities under the LLM, while a seen example is less likely to have words with such low probabilities.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Min-K% Prob detects pretraining data in LLMs by flagging outlier low-probability words in text, achieving 7.4% better performance than prior methods on the new WIKIMIA benchmark.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"cbeff4c1f9e7685aa941265fdbac4461889b75aa027e8579a4782975078bd432"},"source":{"id":"2310.16789","kind":"arxiv","version":3},"verdict":{"id":"78a240a8-5928-4796-90b4-4f4a5990df5c","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-17T18:01:57.778273Z","strongest_claim":"Min-K% Prob achieves a 7.4% improvement on WIKIMIA over these previous methods. We apply Min-K% Prob to three real-world scenarios, copyrighted book detection, contaminated downstream example detection and privacy auditing of machine unlearning, and find it a consistently effective solution.","one_line_summary":"Min-K% Prob detects pretraining data in LLMs by flagging outlier low-probability words in text, achieving 7.4% better performance than prior methods on the new WIKIMIA benchmark.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"An unseen example is likely to contain a few outlier words with low probabilities under the LLM, while a seen example is less likely to have words with such low probabilities.","pith_extraction_headline":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens."},"references":{"count":130,"sample":[{"doi":"","year":2020,"title":"Stability of stochastic gradient descent on nonsmooth convex losses","work_id":"4c7008d6-92a6-49e9-8b4d-63cdeff75cf0","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2023,"title":"Pythia: A suite for analyzing large language models across training and scaling","work_id":"f0f0e465-e8ac-4001-9e07-e6720a687022","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2022,"title":"S., Purohit, S., Reynolds, L., Tow, J., Wang, B., and Weinbach, S","work_id":"168a55d5-675d-49cf-be47-a17ee8cd742e","ref_index":3,"cited_arxiv_id":"2204.06745","is_internal_anchor":true},{"doi":"","year":2021,"title":"Machine unlearning","work_id":"4d425956-f000-49d7-9d36-97aa726d7a61","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":1901,"title":"Language models are few-shot learners","work_id":"6c597039-6bb7-4ed9-9d83-5c66c8dfa4d6","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":130,"snapshot_sha256":"d902b7528a369d6a69e1ba7de7a4aa893b3b9ca3ab21ede50001a7c3980f3010","internal_anchors":7},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"78a240a8-5928-4796-90b4-4f4a5990df5c"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vvEIhE0wyICrjsIEeek4eWGLQ2EGHztCPRU0//v/H4cK+SEgM7V2lM6R3agjSbel/lW7y0sTBbviR1MDmaTOCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-19T02:42:22.637970Z"},"content_sha256":"72e1a730e12159f31c88aab84fbfc1b3325ef6fab1e39bd721d41e3474d8d7e4","schema_version":"1.0","event_id":"sha256:72e1a730e12159f31c88aab84fbfc1b3325ef6fab1e39bd721d41e3474d8d7e4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AVYX74M7QB6UASGVPFJM7RADFX/bundle.json","state_url":"https://pith.science/pith/AVYX74M7QB6UASGVPFJM7RADFX/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AVYX74M7QB6UASGVPFJM7RADFX/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-19T02:42:22Z","links":{"resolver":"https://pith.science/pith/AVYX74M7QB6UASGVPFJM7RADFX","bundle":"https://pith.science/pith/AVYX74M7QB6UASGVPFJM7RADFX/bundle.json","state":"https://pith.science/pith/AVYX74M7QB6UASGVPFJM7RADFX/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AVYX74M7QB6UASGVPFJM7RADFX/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:AVYX74M7QB6UASGVPFJM7RADFX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c0edcfe54c48ba8be05af857532100278a4dad7c829824b0fd3ffbd5584273ec","cross_cats_sorted":["cs.CR","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-25T17:21:23Z","title_canon_sha256":"eab3e6f8bd1ab37ec5727705f226b4a49b4c5d1b21a3097644eb670981474e65"},"schema_version":"1.0","source":{"id":"2310.16789","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.16789","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"arxiv_version","alias_value":"2310.16789v3","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.16789","created_at":"2026-05-17T23:38:13Z"},{"alias_kind":"pith_short_12","alias_value":"AVYX74M7QB6U","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"AVYX74M7QB6UASGV","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"AVYX74M7","created_at":"2026-05-18T12:33:33Z"}],"graph_snapshots":[{"event_id":"sha256:72e1a730e12159f31c88aab84fbfc1b3325ef6fab1e39bd721d41e3474d8d7e4","target":"graph","created_at":"2026-05-17T23:38:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Min-K% Prob achieves a 7.4% improvement on WIKIMIA over these previous methods. We apply Min-K% Prob to three real-world scenarios, copyrighted book detection, contaminated downstream example detection and privacy auditing of machine unlearning, and find it a consistently effective solution."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"An unseen example is likely to contain a few outlier words with low probabilities under the LLM, while a seen example is less likely to have words with such low probabilities."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Min-K% Prob detects pretraining data in LLMs by flagging outlier low-probability words in text, achieving 7.4% better performance than prior methods on the new WIKIMIA benchmark."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens."}],"snapshot_sha256":"cbeff4c1f9e7685aa941265fdbac4461889b75aa027e8579a4782975078bd432"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Although large language models (LLMs) are widely deployed, the data used to train them is rarely disclosed. Given the incredible scale of this data, up to trillions of tokens, it is all but certain that it includes potentially problematic text such as copyrighted materials, personally identifiable information, and test data for widely reported reference benchmarks. However, we currently have no way to know which data of these types is included or in what proportions. In this paper, we study the pretraining data detection problem: given a piece of text and black-box access to an LLM without kno","authors_text":"Anirudh Ajith, Danqi Chen, Daogao Liu, Luke Zettlemoyer, Mengzhou Xia, Terra Blevins, Weijia Shi, Yangsibo Huang","cross_cats":["cs.CR","cs.LG"],"headline":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-25T17:21:23Z","title":"Detecting Pretraining Data from Large Language Models"},"references":{"count":130,"internal_anchors":7,"resolved_work":130,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"Stability of stochastic gradient descent on nonsmooth convex losses","work_id":"4c7008d6-92a6-49e9-8b4d-63cdeff75cf0","year":2020},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Pythia: A suite for analyzing large language models across training and scaling","work_id":"f0f0e465-e8ac-4001-9e07-e6720a687022","year":2023},{"cited_arxiv_id":"2204.06745","doi":"","is_internal_anchor":true,"ref_index":3,"title":"S., Purohit, S., Reynolds, L., Tow, J., Wang, B., and Weinbach, S","work_id":"168a55d5-675d-49cf-be47-a17ee8cd742e","year":2022},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Machine unlearning","work_id":"4d425956-f000-49d7-9d36-97aa726d7a61","year":2021},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Language models are few-shot learners","work_id":"6c597039-6bb7-4ed9-9d83-5c66c8dfa4d6","year":1901}],"snapshot_sha256":"d902b7528a369d6a69e1ba7de7a4aa893b3b9ca3ab21ede50001a7c3980f3010"},"source":{"id":"2310.16789","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-17T18:01:57.778273Z","id":"78a240a8-5928-4796-90b4-4f4a5990df5c","model_set":{"reader":"grok-4.3"},"one_line_summary":"Min-K% Prob detects pretraining data in LLMs by flagging outlier low-probability words in text, achieving 7.4% better performance than prior methods on the new WIKIMIA benchmark.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Min-K% Prob detects if text was in an LLM's pretraining data by averaging the lowest-probability tokens.","strongest_claim":"Min-K% Prob achieves a 7.4% improvement on WIKIMIA over these previous methods. We apply Min-K% Prob to three real-world scenarios, copyrighted book detection, contaminated downstream example detection and privacy auditing of machine unlearning, and find it a consistently effective solution.","weakest_assumption":"An unseen example is likely to contain a few outlier words with low probabilities under the LLM, while a seen example is less likely to have words with such low probabilities."}},"verdict_id":"78a240a8-5928-4796-90b4-4f4a5990df5c"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ac2317ad040080b6d887ab63cb9f95efb9c89c084bec08009290b57d4a5268d2","target":"record","created_at":"2026-05-17T23:38:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c0edcfe54c48ba8be05af857532100278a4dad7c829824b0fd3ffbd5584273ec","cross_cats_sorted":["cs.CR","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-25T17:21:23Z","title_canon_sha256":"eab3e6f8bd1ab37ec5727705f226b4a49b4c5d1b21a3097644eb670981474e65"},"schema_version":"1.0","source":{"id":"2310.16789","kind":"arxiv","version":3}},"canonical_sha256":"05717ff19f807d4048d57952cfc4032dfa072bf49732ab3f13cb7f60ae8cfb4a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"05717ff19f807d4048d57952cfc4032dfa072bf49732ab3f13cb7f60ae8cfb4a","first_computed_at":"2026-05-17T23:38:13.446899Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:13.446899Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"YiGnw1vUnKo9G85xBdtxfQ/JiQg4os6tle9cTxsFYdf67aLj51O97Gj4IYd0U42om5vikFlJdpHf7kQi7S1qAg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:13.447891Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.16789","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ac2317ad040080b6d887ab63cb9f95efb9c89c084bec08009290b57d4a5268d2","sha256:72e1a730e12159f31c88aab84fbfc1b3325ef6fab1e39bd721d41e3474d8d7e4"],"state_sha256":"e3aa0861b5b7a6660ca8bfae8472513e7e57d4cf35ddd124b64c9debce34cbe4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D4DAoKcsVZEewcslHuoKhr8gjuD/hV9xt/Oo3tQMuvrCnDCXALe+ingzjt1pIOQcIyyyFwrsMfYUh3QJpfWYAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-19T02:42:22.639597Z","bundle_sha256":"2ff9f095f3f6cce9394f407d84905180f5eca6ca9b9a2eb31df9280aabb888bf"}}