{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:UAROUVVS2G6BH34ME7VXHZNH4O","short_pith_number":"pith:UAROUVVS","schema_version":"1.0","canonical_sha256":"a022ea56b2d1bc13ef8c27eb73e5a7e39f9d1886ae7f1f28f0a6f2fa2ac87981","source":{"kind":"arxiv","id":"2505.08389","version":1},"attestation_state":"computed","paper":{"title":"Towards Contamination Resistant Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Rahmatullah Musawi, Sheng Lu","submitted_at":"2025-05-13T09:35:40Z","abstract_excerpt":"The rapid development of large language models (LLMs) has transformed the landscape of natural language processing. Evaluating LLMs properly is crucial for understanding their potential and addressing concerns such as safety. However, LLM evaluation is confronted by various factors, among which contamination stands out as a key issue that undermines the reliability of evaluations. In this work, we introduce the concept of contamination resistance to address this challenge. We propose a benchmark based on Caesar ciphers (e.g., \"ab\" to \"bc\" when the shift is 1), which, despite its simplicity, is"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.08389","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T09:35:40Z","cross_cats_sorted":[],"title_canon_sha256":"8cf4213844ee97b78ad2627b3e5003335c46f2b2d6d2a3fd2c2f75333798e331","abstract_canon_sha256":"24a013b90e9381f0ba8d16955787015f9115cbe4040f00df0556296022c45fd6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:02:29.968583Z","signature_b64":"9qEohER1GyCCRpy7Yga3JqonpmOeFZnhPTuayDk7Q9P/tyhMuBJ9nsncZa7qTonuMp4cQfTwBA7soSH/0CS/AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a022ea56b2d1bc13ef8c27eb73e5a7e39f9d1886ae7f1f28f0a6f2fa2ac87981","last_reissued_at":"2026-07-05T11:02:29.968085Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:02:29.968085Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Towards Contamination Resistant Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Rahmatullah Musawi, Sheng Lu","submitted_at":"2025-05-13T09:35:40Z","abstract_excerpt":"The rapid development of large language models (LLMs) has transformed the landscape of natural language processing. Evaluating LLMs properly is crucial for understanding their potential and addressing concerns such as safety. However, LLM evaluation is confronted by various factors, among which contamination stands out as a key issue that undermines the reliability of evaluations. In this work, we introduce the concept of contamination resistance to address this challenge. We propose a benchmark based on Caesar ciphers (e.g., \"ab\" to \"bc\" when the shift is 1), which, despite its simplicity, is"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.08389","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.08389/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.08389","created_at":"2026-07-05T11:02:29.968145+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.08389v1","created_at":"2026-07-05T11:02:29.968145+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.08389","created_at":"2026-07-05T11:02:29.968145+00:00"},{"alias_kind":"pith_short_12","alias_value":"UAROUVVS2G6B","created_at":"2026-07-05T11:02:29.968145+00:00"},{"alias_kind":"pith_short_16","alias_value":"UAROUVVS2G6BH34M","created_at":"2026-07-05T11:02:29.968145+00:00"},{"alias_kind":"pith_short_8","alias_value":"UAROUVVS","created_at":"2026-07-05T11:02:29.968145+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.19999","citing_title":"LLM Benchmark Datasets Should Be Contamination-Resistant","ref_index":15,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O","json":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O.json","graph_json":"https://pith.science/api/pith-number/UAROUVVS2G6BH34ME7VXHZNH4O/graph.json","events_json":"https://pith.science/api/pith-number/UAROUVVS2G6BH34ME7VXHZNH4O/events.json","paper":"https://pith.science/paper/UAROUVVS"},"agent_actions":{"view_html":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O","download_json":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O.json","view_paper":"https://pith.science/paper/UAROUVVS","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.08389&json=true","fetch_graph":"https://pith.science/api/pith-number/UAROUVVS2G6BH34ME7VXHZNH4O/graph.json","fetch_events":"https://pith.science/api/pith-number/UAROUVVS2G6BH34ME7VXHZNH4O/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O/action/storage_attestation","attest_author":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O/action/author_attestation","sign_citation":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O/action/citation_signature","submit_replication":"https://pith.science/pith/UAROUVVS2G6BH34ME7VXHZNH4O/action/replication_record"}},"created_at":"2026-07-05T11:02:29.968145+00:00","updated_at":"2026-07-05T11:02:29.968145+00:00"}