{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:LTLUJA5AKZEBLBAVMYMVMOTQE7","short_pith_number":"pith:LTLUJA5A","canonical_record":{"source":{"id":"2308.11696","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T17:59:30Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"58eb45aec2dc0c6cd9ed6278f6da701c1b1216505c103c49b7a6a4c61f7d7531","abstract_canon_sha256":"28a9e73830da07fb32849c93cc677b49e7a248a10ebbbff0d32471e2bb7e60fd"},"schema_version":"1.0"},"canonical_sha256":"5cd74483a056481584156619563a7027e3b87087edf5f4f381e47df6685bf138","source":{"kind":"arxiv","id":"2308.11696","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.11696","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"arxiv_version","alias_value":"2308.11696v5","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.11696","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_12","alias_value":"LTLUJA5AKZEB","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_16","alias_value":"LTLUJA5AKZEBLBAV","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_8","alias_value":"LTLUJA5A","created_at":"2026-07-05T08:02:45Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:LTLUJA5AKZEBLBAVMYMVMOTQE7","target":"record","payload":{"canonical_record":{"source":{"id":"2308.11696","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T17:59:30Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"58eb45aec2dc0c6cd9ed6278f6da701c1b1216505c103c49b7a6a4c61f7d7531","abstract_canon_sha256":"28a9e73830da07fb32849c93cc677b49e7a248a10ebbbff0d32471e2bb7e60fd"},"schema_version":"1.0"},"canonical_sha256":"5cd74483a056481584156619563a7027e3b87087edf5f4f381e47df6685bf138","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:02:45.669699Z","signature_b64":"l5byeCSxPtRJScUeiabtgGm65BsuUFP+y0YuL1ZySkyb7INDhKkj+feGhHGgB/aiZ/GNOo9mDYb/RaWCmyAYDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5cd74483a056481584156619563a7027e3b87087edf5f4f381e47df6685bf138","last_reissued_at":"2026-07-05T08:02:45.669180Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:02:45.669180Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.11696","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:02:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nE2xZeBpxku173yaj9x/cz70/ps6kqLsrn5xTgiSWzFrLcn7u5ouCvMtxaKxVx4xKHKlkd3ZJkn11VTJn2KZDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T14:23:56.229148Z"},"content_sha256":"3609bfcbd2e5a29d72f071c7939542de976773a12d5ab3c0860e625bcfe53e2d","schema_version":"1.0","event_id":"sha256:3609bfcbd2e5a29d72f071c7939542de976773a12d5ab3c0860e625bcfe53e2d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:LTLUJA5AKZEBLBAVMYMVMOTQE7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Efficient Benchmarking of Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CV","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ariel Gera, Elron Bandel, Eyal Shnarch, Leshem Choshen, Liat Ein-Dor, Michal Shmueli-Scheuer, Noam Slonim, Ofir Arviv, Yotam Perlitz","submitted_at":"2023-08-22T17:59:30Z","abstract_excerpt":"The increasing versatility of language models (LMs) has given rise to a new class of benchmarks that comprehensively assess a broad range of capabilities. Such benchmarks are associated with massive computational costs, extending to thousands of GPU hours per model. However, the efficiency aspect of these evaluation efforts had raised little discussion in the literature. In this work, we present the problem of Efficient Benchmarking, namely, intelligently reducing the computation costs of LM evaluation without compromising reliability. Using the HELM benchmark as a test case, we investigate ho"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.11696","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.11696/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:02:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RKsFV15A8cJbsVyKDdJDy8peX4YLnTJqy0U46wQfIzwsMHHsrmZEPJVeJluqfJPKRorDTNWtLY2cnB+zz/m+AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T14:23:56.229842Z"},"content_sha256":"630af1bc541672783ad3f9222aa722faeb124da32de3565340b4d3c2245ae25e","schema_version":"1.0","event_id":"sha256:630af1bc541672783ad3f9222aa722faeb124da32de3565340b4d3c2245ae25e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/bundle.json","state_url":"https://pith.science/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T14:23:56Z","links":{"resolver":"https://pith.science/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7","bundle":"https://pith.science/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/bundle.json","state":"https://pith.science/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LTLUJA5AKZEBLBAVMYMVMOTQE7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:LTLUJA5AKZEBLBAVMYMVMOTQE7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"28a9e73830da07fb32849c93cc677b49e7a248a10ebbbff0d32471e2bb7e60fd","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T17:59:30Z","title_canon_sha256":"58eb45aec2dc0c6cd9ed6278f6da701c1b1216505c103c49b7a6a4c61f7d7531"},"schema_version":"1.0","source":{"id":"2308.11696","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.11696","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"arxiv_version","alias_value":"2308.11696v5","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.11696","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_12","alias_value":"LTLUJA5AKZEB","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_16","alias_value":"LTLUJA5AKZEBLBAV","created_at":"2026-07-05T08:02:45Z"},{"alias_kind":"pith_short_8","alias_value":"LTLUJA5A","created_at":"2026-07-05T08:02:45Z"}],"graph_snapshots":[{"event_id":"sha256:630af1bc541672783ad3f9222aa722faeb124da32de3565340b4d3c2245ae25e","target":"graph","created_at":"2026-07-05T08:02:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.11696/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The increasing versatility of language models (LMs) has given rise to a new class of benchmarks that comprehensively assess a broad range of capabilities. Such benchmarks are associated with massive computational costs, extending to thousands of GPU hours per model. However, the efficiency aspect of these evaluation efforts had raised little discussion in the literature. In this work, we present the problem of Efficient Benchmarking, namely, intelligently reducing the computation costs of LM evaluation without compromising reliability. Using the HELM benchmark as a test case, we investigate ho","authors_text":"Ariel Gera, Elron Bandel, Eyal Shnarch, Leshem Choshen, Liat Ein-Dor, Michal Shmueli-Scheuer, Noam Slonim, Ofir Arviv, Yotam Perlitz","cross_cats":["cs.AI","cs.CV","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T17:59:30Z","title":"Efficient Benchmarking of Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.11696","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3609bfcbd2e5a29d72f071c7939542de976773a12d5ab3c0860e625bcfe53e2d","target":"record","created_at":"2026-07-05T08:02:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"28a9e73830da07fb32849c93cc677b49e7a248a10ebbbff0d32471e2bb7e60fd","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-08-22T17:59:30Z","title_canon_sha256":"58eb45aec2dc0c6cd9ed6278f6da701c1b1216505c103c49b7a6a4c61f7d7531"},"schema_version":"1.0","source":{"id":"2308.11696","kind":"arxiv","version":5}},"canonical_sha256":"5cd74483a056481584156619563a7027e3b87087edf5f4f381e47df6685bf138","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5cd74483a056481584156619563a7027e3b87087edf5f4f381e47df6685bf138","first_computed_at":"2026-07-05T08:02:45.669180Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:02:45.669180Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"l5byeCSxPtRJScUeiabtgGm65BsuUFP+y0YuL1ZySkyb7INDhKkj+feGhHGgB/aiZ/GNOo9mDYb/RaWCmyAYDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:02:45.669699Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.11696","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3609bfcbd2e5a29d72f071c7939542de976773a12d5ab3c0860e625bcfe53e2d","sha256:630af1bc541672783ad3f9222aa722faeb124da32de3565340b4d3c2245ae25e"],"state_sha256":"2813bb37c11bd500278c9218c8449589ff60e4a6d8932b01ff4f733b321da924"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uw6J1eGKieOtvDcLiiSJlIDGsOpbZ1RbzGWtrLi5zeT7wkie3MyO3ZDxdngqM3p+shcucGm9XEesoQDrtFoSDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T14:23:56.235410Z","bundle_sha256":"7ef29ec2bfe50e2b33acbc9699b037b531b76488b507616d9405d4177e3beaa7"}}