{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:NLVTCPG57RPQJM72Q6YWNP7PSK","short_pith_number":"pith:NLVTCPG5","schema_version":"1.0","canonical_sha256":"6aeb313cddfc5f04b3fa87b166bfef92958d52b500d9616f72151f10a65853c0","source":{"kind":"arxiv","id":"2404.18824","version":1},"attestation_state":"computed","paper":{"title":"Benchmarking Benchmark Leakage in Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Pengfei Liu, Ruijie Xu, Run-Ze Fan, Zengzhi Wang","submitted_at":"2024-04-29T16:05:36Z","abstract_excerpt":"Amid the expanding use of pre-training data, the phenomenon of benchmark dataset leakage has become increasingly prominent, exacerbated by opaque training processes and the often undisclosed inclusion of supervised data in contemporary Large Language Models (LLMs). This issue skews benchmark effectiveness and fosters potentially unfair comparisons, impeding the field's healthy development. To address this, we introduce a detection pipeline utilizing Perplexity and N-gram accuracy, two simple and scalable metrics that gauge a model's prediction precision on benchmark, to identify potential data"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.18824","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-29T16:05:36Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"1834f46bc141b38f01574305be5724d703d2c153463e909aeb2a539e0032598a","abstract_canon_sha256":"e19312d631805334db9720a267753e7fead5a48ba2e42236883c48b09c268015"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:13:21.236561Z","signature_b64":"LCbtu96f3tnWPAle8ENpoH27Bb6iItyD2P49SwRPFRO7UhEQ2xOCzJ1KthKOXrrS4a+EPQT3S8j+f0dnVoQVDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6aeb313cddfc5f04b3fa87b166bfef92958d52b500d9616f72151f10a65853c0","last_reissued_at":"2026-07-05T08:13:21.236079Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:13:21.236079Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Benchmarking Benchmark Leakage in Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Pengfei Liu, Ruijie Xu, Run-Ze Fan, Zengzhi Wang","submitted_at":"2024-04-29T16:05:36Z","abstract_excerpt":"Amid the expanding use of pre-training data, the phenomenon of benchmark dataset leakage has become increasingly prominent, exacerbated by opaque training processes and the often undisclosed inclusion of supervised data in contemporary Large Language Models (LLMs). This issue skews benchmark effectiveness and fosters potentially unfair comparisons, impeding the field's healthy development. To address this, we introduce a detection pipeline utilizing Perplexity and N-gram accuracy, two simple and scalable metrics that gauge a model's prediction precision on benchmark, to identify potential data"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.18824","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.18824/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.18824","created_at":"2026-07-05T08:13:21.236137+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.18824v1","created_at":"2026-07-05T08:13:21.236137+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.18824","created_at":"2026-07-05T08:13:21.236137+00:00"},{"alias_kind":"pith_short_12","alias_value":"NLVTCPG57RPQ","created_at":"2026-07-05T08:13:21.236137+00:00"},{"alias_kind":"pith_short_16","alias_value":"NLVTCPG57RPQJM72","created_at":"2026-07-05T08:13:21.236137+00:00"},{"alias_kind":"pith_short_8","alias_value":"NLVTCPG5","created_at":"2026-07-05T08:13:21.236137+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":20,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.26350","citing_title":"OpenFinGym: A Verifiable Multi-Task Gym Environment for Evaluating Quant Agents","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2606.23313","citing_title":"Uncertainty-based Debiasing and Unlearning for Decontamination","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23965","citing_title":"LGMT: Logic-Grounded Metamorphic Testing for Evaluating the Reasoning Reliability of LLMs","ref_index":56,"is_internal_anchor":false},{"citing_arxiv_id":"2606.17683","citing_title":"Bridging Functional Correctness and Runtime Efficiency Gaps in LLM-Based Code Translation","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2607.01740","citing_title":"Meta-Benchmarks for Financial-Services LLM Evaluation","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11337","citing_title":"Can AI Agents Synthesize Scientific Conclusions?","ref_index":132,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30989","citing_title":"Wait, am I Being Fair? Characterizing Deductive Stereotyping and Mitigating It with Fair-GCG","ref_index":97,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23965","citing_title":"LGMT: Logic-Grounded Metamorphic Testing for Evaluating the Reasoning Reliability of LLMs","ref_index":56,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29815","citing_title":"SrDetection: A Self-Referential Framework for Data Leakage Detection in Code Large Language Models","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22368","citing_title":"VeriScale: Adversarial Test-Suite Scaling for Verifiable Code Generation","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21930","citing_title":"PITMuS: A Tool for Automated Bug Dataset Generation via Source-Level Mutant Reconstruction","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2508.06226","citing_title":"GeoLaux: A Benchmark for Evaluating MLLMs' Geometry Performance on Long-Step Problems Requiring Auxiliary Lines","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2510.15746","citing_title":"LLMs Judge Themselves: A Game-Theoretic Framework for Human-Aligned Evaluation","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2601.02996","citing_title":"Large Reasoning Models Are (Not Yet) Multilingual Latent Reasoners","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2410.07985","citing_title":"Omni-MATH: A Universal Olympiad Level Mathematic Benchmark For Large Language Models","ref_index":72,"is_internal_anchor":false},{"citing_arxiv_id":"2604.22871","citing_title":"AutoRISE: Agent-Driven Strategy Evolution for Red-Teaming Large Language Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.21255","citing_title":"When Agents Look the Same: Quantifying Distillation-Induced Similarity in Tool-Use Behaviors","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17771","citing_title":"SPENCE: A Syntactic Probe for Detecting Contamination in NL2SQL Benchmarks","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15203","citing_title":"MADE: A Living Benchmark for Multi-Label Text Classification with Uncertainty Quantification of Medical Device Adverse Events","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06327","citing_title":"Measuring Evaluation-Context Divergence in Open-Weight LLMs: A Paired-Prompt Protocol with Pilot Evidence of Alignment-Pipeline-Specific Heterogeneity","ref_index":17,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK","json":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK.json","graph_json":"https://pith.science/api/pith-number/NLVTCPG57RPQJM72Q6YWNP7PSK/graph.json","events_json":"https://pith.science/api/pith-number/NLVTCPG57RPQJM72Q6YWNP7PSK/events.json","paper":"https://pith.science/paper/NLVTCPG5"},"agent_actions":{"view_html":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK","download_json":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK.json","view_paper":"https://pith.science/paper/NLVTCPG5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.18824&json=true","fetch_graph":"https://pith.science/api/pith-number/NLVTCPG57RPQJM72Q6YWNP7PSK/graph.json","fetch_events":"https://pith.science/api/pith-number/NLVTCPG57RPQJM72Q6YWNP7PSK/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK/action/timestamp_anchor","attest_storage":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK/action/storage_attestation","attest_author":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK/action/author_attestation","sign_citation":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK/action/citation_signature","submit_replication":"https://pith.science/pith/NLVTCPG57RPQJM72Q6YWNP7PSK/action/replication_record"}},"created_at":"2026-07-05T08:13:21.236137+00:00","updated_at":"2026-07-05T08:13:21.236137+00:00"}