{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:TLFP23C2YS7GNDRFQ2IDVEMPPZ","short_pith_number":"pith:TLFP23C2","canonical_record":{"source":{"id":"2407.21072","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-29T03:37:14Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"24085bf89526a6c2c5ab77b2e0c90960577e0063f50ac940ab18a587346accec","abstract_canon_sha256":"592d49540a47349452af123f97aa2f9618b1c64e81734bebb3dc7b7fcaa3484d"},"schema_version":"1.0"},"canonical_sha256":"9acafd6c5ac4be668e2586903a918f7e77e59ecd2b9b2d0f2f6fc4cf4ec620d5","source":{"kind":"arxiv","id":"2407.21072","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.21072","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"arxiv_version","alias_value":"2407.21072v1","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.21072","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_12","alias_value":"TLFP23C2YS7G","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_16","alias_value":"TLFP23C2YS7GNDRF","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_8","alias_value":"TLFP23C2","created_at":"2026-07-05T08:50:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:TLFP23C2YS7GNDRFQ2IDVEMPPZ","target":"record","payload":{"canonical_record":{"source":{"id":"2407.21072","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-29T03:37:14Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"24085bf89526a6c2c5ab77b2e0c90960577e0063f50ac940ab18a587346accec","abstract_canon_sha256":"592d49540a47349452af123f97aa2f9618b1c64e81734bebb3dc7b7fcaa3484d"},"schema_version":"1.0"},"canonical_sha256":"9acafd6c5ac4be668e2586903a918f7e77e59ecd2b9b2d0f2f6fc4cf4ec620d5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:50:33.605999Z","signature_b64":"Xg9Ld3rCwekTTT6rbuBuRERr95x5bUNuU023+byjaybqw6xs1d14ToOsE3QGNT/tn7B29Y6AxWZ7Z+lg/FxmCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9acafd6c5ac4be668e2586903a918f7e77e59ecd2b9b2d0f2f6fc4cf4ec620d5","last_reissued_at":"2026-07-05T08:50:33.605485Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:50:33.605485Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.21072","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:50:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Fxe2kcIZNvXDJRh+JU1MeIWr3R32BnvhLWeqpaqztPS/iiNPZ6bj16rQcfx33fvXlNYFUvctG7WQfORJU0/aCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T12:09:02.069277Z"},"content_sha256":"ebe69ba000ff65e5421880992106c624fd51d14b633c2c9f229f06d3c7d71688","schema_version":"1.0","event_id":"sha256:ebe69ba000ff65e5421880992106c624fd51d14b633c2c9f229f06d3c7d71688"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:TLFP23C2YS7GNDRFQ2IDVEMPPZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Metrics: A Critical Analysis of the Variability in Large Language Model Evaluation Frameworks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Cl\\'ement Christophe, Marco AF Pimentel, Prateek Munjal, Praveen K Kanithi, Shadab Khan, Tathagata Raha","submitted_at":"2024-07-29T03:37:14Z","abstract_excerpt":"As large language models (LLMs) continue to evolve, the need for robust and standardized evaluation benchmarks becomes paramount. Evaluating the performance of these models is a complex challenge that requires careful consideration of various linguistic tasks, model architectures, and benchmarking methodologies. In recent years, various frameworks have emerged as noteworthy contributions to the field, offering comprehensive evaluation tests and benchmarks for assessing the capabilities of LLMs across diverse domains. This paper provides an exploration and critical analysis of some of these eva"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.21072","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.21072/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:50:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/SpcvbKtXmBHmRnNCuNnXXvVBz7ekYsOaQcIML1xi5y2jmsQtspGEYGzb8FdUAAjWiMRb0UG3SEQffy/ZdJGAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T12:09:02.069671Z"},"content_sha256":"f476df6b6dfb85895d922d2644810cf6f67c126a3ef2e18e0c9e324b76299560","schema_version":"1.0","event_id":"sha256:f476df6b6dfb85895d922d2644810cf6f67c126a3ef2e18e0c9e324b76299560"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/bundle.json","state_url":"https://pith.science/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T12:09:02Z","links":{"resolver":"https://pith.science/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ","bundle":"https://pith.science/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/bundle.json","state":"https://pith.science/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TLFP23C2YS7GNDRFQ2IDVEMPPZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TLFP23C2YS7GNDRFQ2IDVEMPPZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"592d49540a47349452af123f97aa2f9618b1c64e81734bebb3dc7b7fcaa3484d","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-29T03:37:14Z","title_canon_sha256":"24085bf89526a6c2c5ab77b2e0c90960577e0063f50ac940ab18a587346accec"},"schema_version":"1.0","source":{"id":"2407.21072","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.21072","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"arxiv_version","alias_value":"2407.21072v1","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.21072","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_12","alias_value":"TLFP23C2YS7G","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_16","alias_value":"TLFP23C2YS7GNDRF","created_at":"2026-07-05T08:50:33Z"},{"alias_kind":"pith_short_8","alias_value":"TLFP23C2","created_at":"2026-07-05T08:50:33Z"}],"graph_snapshots":[{"event_id":"sha256:f476df6b6dfb85895d922d2644810cf6f67c126a3ef2e18e0c9e324b76299560","target":"graph","created_at":"2026-07-05T08:50:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.21072/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As large language models (LLMs) continue to evolve, the need for robust and standardized evaluation benchmarks becomes paramount. Evaluating the performance of these models is a complex challenge that requires careful consideration of various linguistic tasks, model architectures, and benchmarking methodologies. In recent years, various frameworks have emerged as noteworthy contributions to the field, offering comprehensive evaluation tests and benchmarks for assessing the capabilities of LLMs across diverse domains. This paper provides an exploration and critical analysis of some of these eva","authors_text":"Cl\\'ement Christophe, Marco AF Pimentel, Prateek Munjal, Praveen K Kanithi, Shadab Khan, Tathagata Raha","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-29T03:37:14Z","title":"Beyond Metrics: A Critical Analysis of the Variability in Large Language Model Evaluation Frameworks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.21072","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ebe69ba000ff65e5421880992106c624fd51d14b633c2c9f229f06d3c7d71688","target":"record","created_at":"2026-07-05T08:50:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"592d49540a47349452af123f97aa2f9618b1c64e81734bebb3dc7b7fcaa3484d","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-29T03:37:14Z","title_canon_sha256":"24085bf89526a6c2c5ab77b2e0c90960577e0063f50ac940ab18a587346accec"},"schema_version":"1.0","source":{"id":"2407.21072","kind":"arxiv","version":1}},"canonical_sha256":"9acafd6c5ac4be668e2586903a918f7e77e59ecd2b9b2d0f2f6fc4cf4ec620d5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9acafd6c5ac4be668e2586903a918f7e77e59ecd2b9b2d0f2f6fc4cf4ec620d5","first_computed_at":"2026-07-05T08:50:33.605485Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:50:33.605485Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Xg9Ld3rCwekTTT6rbuBuRERr95x5bUNuU023+byjaybqw6xs1d14ToOsE3QGNT/tn7B29Y6AxWZ7Z+lg/FxmCw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:50:33.605999Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.21072","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ebe69ba000ff65e5421880992106c624fd51d14b633c2c9f229f06d3c7d71688","sha256:f476df6b6dfb85895d922d2644810cf6f67c126a3ef2e18e0c9e324b76299560"],"state_sha256":"af62fc17cf75ddd69a1c8cd034ad83d85d910e11fffe422ddd0381df5dbc8945"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kylTFEKkRQUrCDmxy4u7tCBLhu4KRBawzSeIffAxunWTeivzCojkn2D6+QOnuXT5eX3GHaMnBRgnfZoM4rokBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T12:09:02.072066Z","bundle_sha256":"ec3cdfcf0b2ac8b75bc187da12fdb5ea1e051c182ea62847d0f0d337934cc02b"}}