{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NLVTCPG57RPQJM72Q6YWNP7PSK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e19312d631805334db9720a267753e7fead5a48ba2e42236883c48b09c268015","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-29T16:05:36Z","title_canon_sha256":"1834f46bc141b38f01574305be5724d703d2c153463e909aeb2a539e0032598a"},"schema_version":"1.0","source":{"id":"2404.18824","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.18824","created_at":"2026-07-05T08:13:21Z"},{"alias_kind":"arxiv_version","alias_value":"2404.18824v1","created_at":"2026-07-05T08:13:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.18824","created_at":"2026-07-05T08:13:21Z"},{"alias_kind":"pith_short_12","alias_value":"NLVTCPG57RPQ","created_at":"2026-07-05T08:13:21Z"},{"alias_kind":"pith_short_16","alias_value":"NLVTCPG57RPQJM72","created_at":"2026-07-05T08:13:21Z"},{"alias_kind":"pith_short_8","alias_value":"NLVTCPG5","created_at":"2026-07-05T08:13:21Z"}],"graph_snapshots":[{"event_id":"sha256:ce196a00f64a689d55d27a0ad1280e0dfa48189a42d8471bc195cfbe3fc65b23","target":"graph","created_at":"2026-07-05T08:13:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.18824/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Amid the expanding use of pre-training data, the phenomenon of benchmark dataset leakage has become increasingly prominent, exacerbated by opaque training processes and the often undisclosed inclusion of supervised data in contemporary Large Language Models (LLMs). This issue skews benchmark effectiveness and fosters potentially unfair comparisons, impeding the field's healthy development. To address this, we introduce a detection pipeline utilizing Perplexity and N-gram accuracy, two simple and scalable metrics that gauge a model's prediction precision on benchmark, to identify potential data","authors_text":"Pengfei Liu, Ruijie Xu, Run-Ze Fan, Zengzhi Wang","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-29T16:05:36Z","title":"Benchmarking Benchmark Leakage in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.18824","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:09f294fd8eb73ab116c3f5c6ca85b545596886acd33157f28403dfef94b838bc","target":"record","created_at":"2026-07-05T08:13:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e19312d631805334db9720a267753e7fead5a48ba2e42236883c48b09c268015","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-29T16:05:36Z","title_canon_sha256":"1834f46bc141b38f01574305be5724d703d2c153463e909aeb2a539e0032598a"},"schema_version":"1.0","source":{"id":"2404.18824","kind":"arxiv","version":1}},"canonical_sha256":"6aeb313cddfc5f04b3fa87b166bfef92958d52b500d9616f72151f10a65853c0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6aeb313cddfc5f04b3fa87b166bfef92958d52b500d9616f72151f10a65853c0","first_computed_at":"2026-07-05T08:13:21.236079Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:13:21.236079Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LCbtu96f3tnWPAle8ENpoH27Bb6iItyD2P49SwRPFRO7UhEQ2xOCzJ1KthKOXrrS4a+EPQT3S8j+f0dnVoQVDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:13:21.236561Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.18824","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:09f294fd8eb73ab116c3f5c6ca85b545596886acd33157f28403dfef94b838bc","sha256:ce196a00f64a689d55d27a0ad1280e0dfa48189a42d8471bc195cfbe3fc65b23"],"state_sha256":"1ff833db6c654bf300a069107506c888d249000a9165bfb8f1455aaed3e34a5a"}