{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:WE255WASGWL6QVPJIW63VQX4AV","short_pith_number":"pith:WE255WAS","canonical_record":{"source":{"id":"2412.03597","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T20:49:21Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"32513eeed06c060390a6bdb2aa896248500b9f531472cf2063a3802db2b716b9","abstract_canon_sha256":"969e24221cd988f4aba4ee5728ce5135c22cb96079684bf2e828e62f600dd027"},"schema_version":"1.0"},"canonical_sha256":"b135ded8123597e855e945bdbac2fc05480f55076a356d570a3bcc289c14e765","source":{"kind":"arxiv","id":"2412.03597","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.03597","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"arxiv_version","alias_value":"2412.03597v1","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03597","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_12","alias_value":"WE255WASGWL6","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_16","alias_value":"WE255WASGWL6QVPJ","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_8","alias_value":"WE255WAS","created_at":"2026-07-05T09:44:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:WE255WASGWL6QVPJIW63VQX4AV","target":"record","payload":{"canonical_record":{"source":{"id":"2412.03597","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T20:49:21Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"32513eeed06c060390a6bdb2aa896248500b9f531472cf2063a3802db2b716b9","abstract_canon_sha256":"969e24221cd988f4aba4ee5728ce5135c22cb96079684bf2e828e62f600dd027"},"schema_version":"1.0"},"canonical_sha256":"b135ded8123597e855e945bdbac2fc05480f55076a356d570a3bcc289c14e765","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:44:38.487999Z","signature_b64":"a00T+IaMPjf02+vS14UvQqjhYR9ZUvQeOGrNqvipmk2fyOT90SVaAS44TFVF2xNtCAco+2ZwqPU/teB30kYNDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b135ded8123597e855e945bdbac2fc05480f55076a356d570a3bcc289c14e765","last_reissued_at":"2026-07-05T09:44:38.487520Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:44:38.487520Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.03597","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CTj5g23xzMXrJ2qsj9ZVsm/GXZUKY/2eTs5BpAU3BtTRA3zZgagcNx0mNW1Cka0x8rKsSwqqQ7r/eunoVkTbDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T18:43:12.440502Z"},"content_sha256":"2b75fb1cdb3a9ac926101c827e0a2244319306eee83e1a49a763e14913ffa3b2","schema_version":"1.0","event_id":"sha256:2b75fb1cdb3a9ac926101c827e0a2244319306eee83e1a49a763e14913ffa3b2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:WE255WASGWL6QVPJIW63VQX4AV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Vulnerability of Language Model Benchmarks: Do They Accurately Reflect True LLM Performance?","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.LG","stat.ML"],"primary_cat":"cs.CL","authors_text":"Ayushi Agarwal, Eishkaran Singh, Sourav Banerjee","submitted_at":"2024-12-02T20:49:21Z","abstract_excerpt":"The pursuit of leaderboard rankings in Large Language Models (LLMs) has created a fundamental paradox: models excel at standardized tests while failing to demonstrate genuine language understanding and adaptability. Our systematic analysis of NLP evaluation frameworks reveals pervasive vulnerabilities across the evaluation spectrum, from basic metrics to complex benchmarks like GLUE and MMLU. These vulnerabilities manifest through benchmark exploitation, dataset contamination, and evaluation bias, creating a false perception of progress in language understanding capabilities. Through extensive"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03597","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.03597/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"osbRX8gojtT3hTmT4PqC4dnmtI4c7DbpxvuocQwwIEt4g62NklTwyxUS4trcWW2dKQ08IDaG2+JZ48fvaIK0Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T18:43:12.441028Z"},"content_sha256":"8655aafb83a9c074241c83487594347b24bfa578054c92d5cd7b49a8820440ee","schema_version":"1.0","event_id":"sha256:8655aafb83a9c074241c83487594347b24bfa578054c92d5cd7b49a8820440ee"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WE255WASGWL6QVPJIW63VQX4AV/bundle.json","state_url":"https://pith.science/pith/WE255WASGWL6QVPJIW63VQX4AV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WE255WASGWL6QVPJIW63VQX4AV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T18:43:12Z","links":{"resolver":"https://pith.science/pith/WE255WASGWL6QVPJIW63VQX4AV","bundle":"https://pith.science/pith/WE255WASGWL6QVPJIW63VQX4AV/bundle.json","state":"https://pith.science/pith/WE255WASGWL6QVPJIW63VQX4AV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WE255WASGWL6QVPJIW63VQX4AV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WE255WASGWL6QVPJIW63VQX4AV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"969e24221cd988f4aba4ee5728ce5135c22cb96079684bf2e828e62f600dd027","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T20:49:21Z","title_canon_sha256":"32513eeed06c060390a6bdb2aa896248500b9f531472cf2063a3802db2b716b9"},"schema_version":"1.0","source":{"id":"2412.03597","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.03597","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"arxiv_version","alias_value":"2412.03597v1","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03597","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_12","alias_value":"WE255WASGWL6","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_16","alias_value":"WE255WASGWL6QVPJ","created_at":"2026-07-05T09:44:38Z"},{"alias_kind":"pith_short_8","alias_value":"WE255WAS","created_at":"2026-07-05T09:44:38Z"}],"graph_snapshots":[{"event_id":"sha256:8655aafb83a9c074241c83487594347b24bfa578054c92d5cd7b49a8820440ee","target":"graph","created_at":"2026-07-05T09:44:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.03597/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The pursuit of leaderboard rankings in Large Language Models (LLMs) has created a fundamental paradox: models excel at standardized tests while failing to demonstrate genuine language understanding and adaptability. Our systematic analysis of NLP evaluation frameworks reveals pervasive vulnerabilities across the evaluation spectrum, from basic metrics to complex benchmarks like GLUE and MMLU. These vulnerabilities manifest through benchmark exploitation, dataset contamination, and evaluation bias, creating a false perception of progress in language understanding capabilities. Through extensive","authors_text":"Ayushi Agarwal, Eishkaran Singh, Sourav Banerjee","cross_cats":["cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T20:49:21Z","title":"The Vulnerability of Language Model Benchmarks: Do They Accurately Reflect True LLM Performance?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03597","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2b75fb1cdb3a9ac926101c827e0a2244319306eee83e1a49a763e14913ffa3b2","target":"record","created_at":"2026-07-05T09:44:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"969e24221cd988f4aba4ee5728ce5135c22cb96079684bf2e828e62f600dd027","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T20:49:21Z","title_canon_sha256":"32513eeed06c060390a6bdb2aa896248500b9f531472cf2063a3802db2b716b9"},"schema_version":"1.0","source":{"id":"2412.03597","kind":"arxiv","version":1}},"canonical_sha256":"b135ded8123597e855e945bdbac2fc05480f55076a356d570a3bcc289c14e765","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b135ded8123597e855e945bdbac2fc05480f55076a356d570a3bcc289c14e765","first_computed_at":"2026-07-05T09:44:38.487520Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:44:38.487520Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"a00T+IaMPjf02+vS14UvQqjhYR9ZUvQeOGrNqvipmk2fyOT90SVaAS44TFVF2xNtCAco+2ZwqPU/teB30kYNDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:44:38.487999Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.03597","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2b75fb1cdb3a9ac926101c827e0a2244319306eee83e1a49a763e14913ffa3b2","sha256:8655aafb83a9c074241c83487594347b24bfa578054c92d5cd7b49a8820440ee"],"state_sha256":"e9e74532fb3c590dd179fa22e21975296b4b7d6e590e7971c2dd246684bd529a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"arFQNjiamguzq8CJN+DqkPgFM5pHOM+M01d6QXq9OVb85/FEcgzj8VBDaoIvdffFC93eQf1GBPplelP2S9XuBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T18:43:12.445028Z","bundle_sha256":"0e0296264462cf662425d38078cbea547c2b5ad398f4aaa1ba75425c70cabe53"}}