{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:3V2SIALTAX55C7FZ6XAMRO7IDG","short_pith_number":"pith:3V2SIALT","canonical_record":{"source":{"id":"2402.09880","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","cross_cats_sorted":["cs.CL","cs.CY","cs.HC"],"title_canon_sha256":"b1ca92b726ec0dcde7f7934ce2a7fdecc3a983bb3bb54ff21427c67bfa274c24","abstract_canon_sha256":"0dd8476eb1f3be85ece828e64ac4c876be24d48939ea53472a5564494a35d1c0"},"schema_version":"1.0"},"canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","source":{"kind":"arxiv","id":"2402.09880","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.09880","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"arxiv_version","alias_value":"2402.09880v2","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.09880","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_12","alias_value":"3V2SIALTAX55","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_16","alias_value":"3V2SIALTAX55C7FZ","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_8","alias_value":"3V2SIALT","created_at":"2026-07-05T11:02:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:3V2SIALTAX55C7FZ6XAMRO7IDG","target":"record","payload":{"canonical_record":{"source":{"id":"2402.09880","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","cross_cats_sorted":["cs.CL","cs.CY","cs.HC"],"title_canon_sha256":"b1ca92b726ec0dcde7f7934ce2a7fdecc3a983bb3bb54ff21427c67bfa274c24","abstract_canon_sha256":"0dd8476eb1f3be85ece828e64ac4c876be24d48939ea53472a5564494a35d1c0"},"schema_version":"1.0"},"canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:02:38.114053Z","signature_b64":"/ANVKlWW9IMLOc45wgbieLP+RP3HNeGclv1TPQ1zZPKAUK7lHrd2J3aRTGDyDAHP2/ZScrdYpd2CtLNedi1RBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","last_reissued_at":"2026-07-05T11:02:38.112961Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:02:38.112961Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.09880","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:02:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ahs1Ifi5fSP2q/xI2ZeuHoDa9qcCe0fhe/YfmhFlJKCv9iyPSuzZ2V/xzeEQ4WjL1f3QdI+ixnwE4t+7FkBOCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T15:08:35.370213Z"},"content_sha256":"5fe7ffadc1577605062f13315183fd491a2dddbe27872f6b3e4b3f83991d11bf","schema_version":"1.0","event_id":"sha256:5fe7ffadc1577605062f13315183fd491a2dddbe27872f6b3e4b3f83991d11bf"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:3V2SIALTAX55C7FZ6XAMRO7IDG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL","cs.CY","cs.HC"],"primary_cat":"cs.AI","authors_text":"Malka N. Halgamuge, Nalin Arachchilage, Paul Watters, Teo Susnjak, Timothy R. McIntosh, Tong Liu","submitted_at":"2024-02-15T11:08:10Z","abstract_excerpt":"The rapid rise in popularity of Large Language Models (LLMs) with emerging capabilities has spurred public curiosity to evaluate and compare different LLMs, leading many researchers to propose their own LLM benchmarks. Noticing preliminary inadequacies in those benchmarks, we embarked on a study to critically assess 23 state-of-the-art LLM benchmarks, using our novel unified evaluation framework through the lenses of people, process, and technology, under the pillars of benchmark functionality and integrity. Our research uncovered significant limitations, including biases, difficulties in meas"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.09880","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.09880/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:02:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"P34ieDdRaaJHVHVt1mEE/aDHYaAV0IAEg/8JvHonJWLGUuZzdwOx6XOnSOIiSe8WYsLN8UXALMMM1MZqMzB6CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T15:08:35.370692Z"},"content_sha256":"f64b2a46fbadcc4020c7143f3ae026aebae0f54b07a112aa49f303993bbfe2ed","schema_version":"1.0","event_id":"sha256:f64b2a46fbadcc4020c7143f3ae026aebae0f54b07a112aa49f303993bbfe2ed"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/bundle.json","state_url":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T15:08:35Z","links":{"resolver":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG","bundle":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/bundle.json","state":"https://pith.science/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3V2SIALTAX55C7FZ6XAMRO7IDG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3V2SIALTAX55C7FZ6XAMRO7IDG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0dd8476eb1f3be85ece828e64ac4c876be24d48939ea53472a5564494a35d1c0","cross_cats_sorted":["cs.CL","cs.CY","cs.HC"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","title_canon_sha256":"b1ca92b726ec0dcde7f7934ce2a7fdecc3a983bb3bb54ff21427c67bfa274c24"},"schema_version":"1.0","source":{"id":"2402.09880","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.09880","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"arxiv_version","alias_value":"2402.09880v2","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.09880","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_12","alias_value":"3V2SIALTAX55","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_16","alias_value":"3V2SIALTAX55C7FZ","created_at":"2026-07-05T11:02:38Z"},{"alias_kind":"pith_short_8","alias_value":"3V2SIALT","created_at":"2026-07-05T11:02:38Z"}],"graph_snapshots":[{"event_id":"sha256:f64b2a46fbadcc4020c7143f3ae026aebae0f54b07a112aa49f303993bbfe2ed","target":"graph","created_at":"2026-07-05T11:02:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.09880/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The rapid rise in popularity of Large Language Models (LLMs) with emerging capabilities has spurred public curiosity to evaluate and compare different LLMs, leading many researchers to propose their own LLM benchmarks. Noticing preliminary inadequacies in those benchmarks, we embarked on a study to critically assess 23 state-of-the-art LLM benchmarks, using our novel unified evaluation framework through the lenses of people, process, and technology, under the pillars of benchmark functionality and integrity. Our research uncovered significant limitations, including biases, difficulties in meas","authors_text":"Malka N. Halgamuge, Nalin Arachchilage, Paul Watters, Teo Susnjak, Timothy R. McIntosh, Tong Liu","cross_cats":["cs.CL","cs.CY","cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","title":"Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.09880","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5fe7ffadc1577605062f13315183fd491a2dddbe27872f6b3e4b3f83991d11bf","target":"record","created_at":"2026-07-05T11:02:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0dd8476eb1f3be85ece828e64ac4c876be24d48939ea53472a5564494a35d1c0","cross_cats_sorted":["cs.CL","cs.CY","cs.HC"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-15T11:08:10Z","title_canon_sha256":"b1ca92b726ec0dcde7f7934ce2a7fdecc3a983bb3bb54ff21427c67bfa274c24"},"schema_version":"1.0","source":{"id":"2402.09880","kind":"arxiv","version":2}},"canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dd7524017305fbd17cb9f5c0c8bbe819bc7c47c39912b847ef85486dbdf68d7a","first_computed_at":"2026-07-05T11:02:38.112961Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:02:38.112961Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/ANVKlWW9IMLOc45wgbieLP+RP3HNeGclv1TPQ1zZPKAUK7lHrd2J3aRTGDyDAHP2/ZScrdYpd2CtLNedi1RBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:02:38.114053Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.09880","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5fe7ffadc1577605062f13315183fd491a2dddbe27872f6b3e4b3f83991d11bf","sha256:f64b2a46fbadcc4020c7143f3ae026aebae0f54b07a112aa49f303993bbfe2ed"],"state_sha256":"160072da6343bbea9948fb4d8607ebca11f5ff684027bd818e15f5d1c01913d8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8FEAJMo58YsEQzsoL1mDkTizRN2bIw6Y2D5Vzo/UVEXPGyBrfuSByXFrb8MXd9bPTR1oYF/o+zpT5JnQ4SwnAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T15:08:35.376240Z","bundle_sha256":"ebdbf4974a74d3d0bfce85365c4b3b2ca33ef4855eac99e6710907711ca84a5a"}}