{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:3V2SIALTAX55C7FZ6XAMRO7IDG","short_pith_number":"pith:3V2SIALT","schema_version":"1.0","canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","source":{"kind":"arxiv","id":"2402.09880","version":2},"attestation_state":"computed","paper":{"title":"Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL","cs.CY","cs.HC"],"primary_cat":"cs.AI","authors_text":"Malka N. Halgamuge, Nalin Arachchilage, Paul Watters, Teo Susnjak, Timothy R. McIntosh, Tong Liu","submitted_at":"2024-02-15T11:08:10Z","abstract_excerpt":"The rapid rise in popularity of Large Language Models (LLMs) with emerging capabilities has spurred public curiosity to evaluate and compare different LLMs, leading many researchers to propose their own LLM benchmarks. Noticing preliminary inadequacies in those benchmarks, we embarked on a study to critically assess 23 state-of-the-art LLM benchmarks, using our novel unified evaluation framework through the lenses of people, process, and technology, under the pillars of benchmark functionality and integrity. Our research uncovered significant limitations, including biases, difficulties in meas"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.09880","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","cross_cats_sorted":["cs.CL","cs.CY","cs.HC"],"title_canon_sha256":"b1ca92b726ec0dcde7f7934ce2a7fdecc3a983bb3bb54ff21427c67bfa274c24","abstract_canon_sha256":"0dd8476eb1f3be85ece828e64ac4c876be24d48939ea53472a5564494a35d1c0"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:02:38.114053Z","signature_b64":"/ANVKlWW9IMLOc45wgbieLP+RP3HNeGclv1TPQ1zZPKAUK7lHrd2J3aRTGDyDAHP2/ZScrdYpd2CtLNedi1RBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","last_reissued_at":"2026-07-05T11:02:38.112961Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:02:38.112961Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL","cs.CY","cs.HC"],"primary_cat":"cs.AI","authors_text":"Malka N. Halgamuge, Nalin Arachchilage, Paul Watters, Teo Susnjak, Timothy R. McIntosh, Tong Liu","submitted_at":"2024-02-15T11:08:10Z","abstract_excerpt":"The rapid rise in popularity of Large Language Models (LLMs) with emerging capabilities has spurred public curiosity to evaluate and compare different LLMs, leading many researchers to propose their own LLM benchmarks. Noticing preliminary inadequacies in those benchmarks, we embarked on a study to critically assess 23 state-of-the-art LLM benchmarks, using our novel unified evaluation framework through the lenses of people, process, and technology, under the pillars of benchmark functionality and integrity. Our research uncovered significant limitations, including biases, difficulties in meas"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.09880","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.09880/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.09880","created_at":"2026-07-05T11:02:38.113478+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.09880v2","created_at":"2026-07-05T11:02:38.113478+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.09880","created_at":"2026-07-05T11:02:38.113478+00:00"},{"alias_kind":"pith_short_12","alias_value":"3V2SIALTAX55","created_at":"2026-07-05T11:02:38.113478+00:00"},{"alias_kind":"pith_short_16","alias_value":"3V2SIALTAX55C7FZ","created_at":"2026-07-05T11:02:38.113478+00:00"},{"alias_kind":"pith_short_8","alias_value":"3V2SIALT","created_at":"2026-07-05T11:02:38.113478+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2406.04244","citing_title":"Benchmark Data Contamination of Large Language Models: A Survey","ref_index":103,"is_internal_anchor":false},{"citing_arxiv_id":"2511.02627","citing_title":"DecompSR: A dataset for decomposed analyses of compositional multihop spatial reasoning","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2501.14249","citing_title":"Humanity's Last Exam","ref_index":35,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG","json":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG.json","graph_json":"https://pith.science/api/pith-number/3V2SIALTAX55C7FZ6XAMRO7IDG/graph.json","events_json":"https://pith.science/api/pith-number/3V2SIALTAX55C7FZ6XAMRO7IDG/events.json","paper":"https://pith.science/paper/3V2SIALT"},"agent_actions":{"view_html":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG","download_json":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG.json","view_paper":"https://pith.science/paper/3V2SIALT","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.09880&json=true","fetch_graph":"https://pith.science/api/pith-number/3V2SIALTAX55C7FZ6XAMRO7IDG/graph.json","fetch_events":"https://pith.science/api/pith-number/3V2SIALTAX55C7FZ6XAMRO7IDG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/action/storage_attestation","attest_author":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/action/author_attestation","sign_citation":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/action/citation_signature","submit_replication":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/action/replication_record"}},"created_at":"2026-07-05T11:02:38.113478+00:00","updated_at":"2026-07-05T11:02:38.113478+00:00"}