{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:NFINKWPZIANG5R4SRLHIKCQ7AE","short_pith_number":"pith:NFINKWPZ","canonical_record":{"source":{"id":"2402.14992","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T22:05:23Z","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"title_canon_sha256":"998752a9a7397a06b73ea1eef92fccbf60efe16edd4eddcfe014659eb34c9a28","abstract_canon_sha256":"3cd6f2d246df5ca2f269d39f2b258eb4dd3cf56d76ccfe9a5d071c535f18b3f8"},"schema_version":"1.0"},"canonical_sha256":"6950d559f9401a6ec7928ace850a1f0101e688cce2cc3d0a245fb49087949def","source":{"kind":"arxiv","id":"2402.14992","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.14992","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"arxiv_version","alias_value":"2402.14992v2","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.14992","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_12","alias_value":"NFINKWPZIANG","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_16","alias_value":"NFINKWPZIANG5R4S","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_8","alias_value":"NFINKWPZ","created_at":"2026-07-05T08:23:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:NFINKWPZIANG5R4SRLHIKCQ7AE","target":"record","payload":{"canonical_record":{"source":{"id":"2402.14992","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T22:05:23Z","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"title_canon_sha256":"998752a9a7397a06b73ea1eef92fccbf60efe16edd4eddcfe014659eb34c9a28","abstract_canon_sha256":"3cd6f2d246df5ca2f269d39f2b258eb4dd3cf56d76ccfe9a5d071c535f18b3f8"},"schema_version":"1.0"},"canonical_sha256":"6950d559f9401a6ec7928ace850a1f0101e688cce2cc3d0a245fb49087949def","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:23:18.519647Z","signature_b64":"v4wNck+ptam/Lk7c0H4XawumrxDgRisbPoLhq8ssKFMFGvVwTykE0vfamJdh0RqqAOtpJrPmGmZLc8pmveBaBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6950d559f9401a6ec7928ace850a1f0101e688cce2cc3d0a245fb49087949def","last_reissued_at":"2026-07-05T08:23:18.519177Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:23:18.519177Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.14992","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:23:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hlp05nOzAcps3SkFnXDp64XrYL0EFplGeMn7lj51+fgikq3oMimLTTkbDvdxLRRyTR8HH+4U+pBTPpqj5v1ZAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:08:26.441574Z"},"content_sha256":"a364f177c6197b8cf8be2e6ac3209459228d53b7138d79e900133c68767eeacf","schema_version":"1.0","event_id":"sha256:a364f177c6197b8cf8be2e6ac3209459228d53b7138d79e900133c68767eeacf"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:NFINKWPZIANG5R4SRLHIKCQ7AE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"tinyBenchmarks: evaluating LLMs with fewer examples","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","stat.ML"],"primary_cat":"cs.CL","authors_text":"Felipe Maia Polo, Gongjun Xu, Leshem Choshen, Lucas Weber, Mikhail Yurochkin, Yuekai Sun","submitted_at":"2024-02-22T22:05:23Z","abstract_excerpt":"The versatility of large language models (LLMs) led to the creation of diverse benchmarks that thoroughly test a variety of language models' abilities. These benchmarks consist of tens of thousands of examples making evaluation of LLMs very expensive. In this paper, we investigate strategies to reduce the number of evaluations needed to assess the performance of an LLM on several key benchmarks. For example, we show that to accurately estimate the performance of an LLM on MMLU, a popular multiple-choice QA benchmark consisting of 14K examples, it is sufficient to evaluate this LLM on 100 curat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.14992","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.14992/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:23:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hrqcjUWZI7yU0/qP95VynlXCVohdEK2YvFPA/DlGPupb8OiOIyJq0qnOAaB1RTb+9uvyHCpmPIw9HJBdB2pqAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T11:08:26.441952Z"},"content_sha256":"0dfa46bbba97889cecb6e3374cecfd0efe3ea7abd74c28f42410efd5ab9415c0","schema_version":"1.0","event_id":"sha256:0dfa46bbba97889cecb6e3374cecfd0efe3ea7abd74c28f42410efd5ab9415c0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/bundle.json","state_url":"https://pith.science/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T11:08:26Z","links":{"resolver":"https://pith.science/pith/NFINKWPZIANG5R4SRLHIKCQ7AE","bundle":"https://pith.science/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/bundle.json","state":"https://pith.science/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NFINKWPZIANG5R4SRLHIKCQ7AE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NFINKWPZIANG5R4SRLHIKCQ7AE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3cd6f2d246df5ca2f269d39f2b258eb4dd3cf56d76ccfe9a5d071c535f18b3f8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T22:05:23Z","title_canon_sha256":"998752a9a7397a06b73ea1eef92fccbf60efe16edd4eddcfe014659eb34c9a28"},"schema_version":"1.0","source":{"id":"2402.14992","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.14992","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"arxiv_version","alias_value":"2402.14992v2","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.14992","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_12","alias_value":"NFINKWPZIANG","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_16","alias_value":"NFINKWPZIANG5R4S","created_at":"2026-07-05T08:23:18Z"},{"alias_kind":"pith_short_8","alias_value":"NFINKWPZ","created_at":"2026-07-05T08:23:18Z"}],"graph_snapshots":[{"event_id":"sha256:0dfa46bbba97889cecb6e3374cecfd0efe3ea7abd74c28f42410efd5ab9415c0","target":"graph","created_at":"2026-07-05T08:23:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.14992/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The versatility of large language models (LLMs) led to the creation of diverse benchmarks that thoroughly test a variety of language models' abilities. These benchmarks consist of tens of thousands of examples making evaluation of LLMs very expensive. In this paper, we investigate strategies to reduce the number of evaluations needed to assess the performance of an LLM on several key benchmarks. For example, we show that to accurately estimate the performance of an LLM on MMLU, a popular multiple-choice QA benchmark consisting of 14K examples, it is sufficient to evaluate this LLM on 100 curat","authors_text":"Felipe Maia Polo, Gongjun Xu, Leshem Choshen, Lucas Weber, Mikhail Yurochkin, Yuekai Sun","cross_cats":["cs.AI","cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T22:05:23Z","title":"tinyBenchmarks: evaluating LLMs with fewer examples"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.14992","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a364f177c6197b8cf8be2e6ac3209459228d53b7138d79e900133c68767eeacf","target":"record","created_at":"2026-07-05T08:23:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3cd6f2d246df5ca2f269d39f2b258eb4dd3cf56d76ccfe9a5d071c535f18b3f8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-22T22:05:23Z","title_canon_sha256":"998752a9a7397a06b73ea1eef92fccbf60efe16edd4eddcfe014659eb34c9a28"},"schema_version":"1.0","source":{"id":"2402.14992","kind":"arxiv","version":2}},"canonical_sha256":"6950d559f9401a6ec7928ace850a1f0101e688cce2cc3d0a245fb49087949def","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6950d559f9401a6ec7928ace850a1f0101e688cce2cc3d0a245fb49087949def","first_computed_at":"2026-07-05T08:23:18.519177Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:23:18.519177Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"v4wNck+ptam/Lk7c0H4XawumrxDgRisbPoLhq8ssKFMFGvVwTykE0vfamJdh0RqqAOtpJrPmGmZLc8pmveBaBA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:23:18.519647Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.14992","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a364f177c6197b8cf8be2e6ac3209459228d53b7138d79e900133c68767eeacf","sha256:0dfa46bbba97889cecb6e3374cecfd0efe3ea7abd74c28f42410efd5ab9415c0"],"state_sha256":"fc7772399b149e52a269fbbe2eced71921e2899bd9bfa687a065b2bf003f81f2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tBDFuBgwcaJ+WQY2zRHv+pjaItl8Rc5K526NXU+l7eJsrwGs30N5LjWA5Rj7g4oU0/1r1JfzmZ6s4s8x7F85Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T11:08:26.443923Z","bundle_sha256":"604e1aa27a1e1b142ad19440e542da13cea1c1dbab9ef87f57066cfd1eaecaa5"}}