{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:IPGUHE4XYCIPNENIHX26DN22PP","short_pith_number":"pith:IPGUHE4X","schema_version":"1.0","canonical_sha256":"43cd439397c090f691a83df5e1b75a7be73f6250cb224a1007f83dbda5c20ed5","source":{"kind":"arxiv","id":"2407.06172","version":3},"attestation_state":"computed","paper":{"title":"On Speeding Up Language Model Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Carla P. Gomes, Christian K. Belardi, Jin Peng Zhou, Kilian Q. Weinberger, Ruihan Wu, Travis Zhang, Wen Sun","submitted_at":"2024-07-08T17:48:42Z","abstract_excerpt":"Developing prompt-based methods with Large Language Models (LLMs) requires making numerous decisions, which give rise to a combinatorial search problem over hyper-parameters. This exhaustive evaluation can be time-consuming and costly. In this paper, we propose an $\\textit{adaptive}$ approach to explore this space. We are exploiting the fact that often only few samples are needed to identify clearly superior or inferior settings, and that many evaluation tests are highly correlated. We lean on multi-armed bandits to sequentially identify the next (method, validation sample)-pair to evaluate an"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2407.06172","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-07-08T17:48:42Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"64430a7a1b732e41a691d405161cf397ec105869bd6cb8d87150924e20a85552","abstract_canon_sha256":"2e5514d0bf7f8eb2b6339173987d4ab94ddb414d5cf4a00e8ff0050f51808072"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:20:38.394010Z","signature_b64":"+KptroaXhrb99NL9ha684MPa66cfc9Hno5pKhFHoa5o5pXS7s/SrhKv1AhNOAAyH8wOMsN3sZQrVo4hRnMzPAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"43cd439397c090f691a83df5e1b75a7be73f6250cb224a1007f83dbda5c20ed5","last_reissued_at":"2026-07-05T10:20:38.393479Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:20:38.393479Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"On Speeding Up Language Model Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Carla P. Gomes, Christian K. Belardi, Jin Peng Zhou, Kilian Q. Weinberger, Ruihan Wu, Travis Zhang, Wen Sun","submitted_at":"2024-07-08T17:48:42Z","abstract_excerpt":"Developing prompt-based methods with Large Language Models (LLMs) requires making numerous decisions, which give rise to a combinatorial search problem over hyper-parameters. This exhaustive evaluation can be time-consuming and costly. In this paper, we propose an $\\textit{adaptive}$ approach to explore this space. We are exploiting the fact that often only few samples are needed to identify clearly superior or inferior settings, and that many evaluation tests are highly correlated. We lean on multi-armed bandits to sequentially identify the next (method, validation sample)-pair to evaluate an"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.06172","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.06172/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2407.06172","created_at":"2026-07-05T10:20:38.393547+00:00"},{"alias_kind":"arxiv_version","alias_value":"2407.06172v3","created_at":"2026-07-05T10:20:38.393547+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.06172","created_at":"2026-07-05T10:20:38.393547+00:00"},{"alias_kind":"pith_short_12","alias_value":"IPGUHE4XYCIP","created_at":"2026-07-05T10:20:38.393547+00:00"},{"alias_kind":"pith_short_16","alias_value":"IPGUHE4XYCIPNENI","created_at":"2026-07-05T10:20:38.393547+00:00"},{"alias_kind":"pith_short_8","alias_value":"IPGUHE4X","created_at":"2026-07-05T10:20:38.393547+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.24020","citing_title":"You Don't Need to Run Every Eval","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2604.06296","citing_title":"AgentOpt v0.1 Technical Report: Client-Side Optimization for LLM-Based Agent","ref_index":31,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP","json":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP.json","graph_json":"https://pith.science/api/pith-number/IPGUHE4XYCIPNENIHX26DN22PP/graph.json","events_json":"https://pith.science/api/pith-number/IPGUHE4XYCIPNENIHX26DN22PP/events.json","paper":"https://pith.science/paper/IPGUHE4X"},"agent_actions":{"view_html":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP","download_json":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP.json","view_paper":"https://pith.science/paper/IPGUHE4X","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2407.06172&json=true","fetch_graph":"https://pith.science/api/pith-number/IPGUHE4XYCIPNENIHX26DN22PP/graph.json","fetch_events":"https://pith.science/api/pith-number/IPGUHE4XYCIPNENIHX26DN22PP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP/action/storage_attestation","attest_author":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP/action/author_attestation","sign_citation":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP/action/citation_signature","submit_replication":"https://pith.science/pith/IPGUHE4XYCIPNENIHX26DN22PP/action/replication_record"}},"created_at":"2026-07-05T10:20:38.393547+00:00","updated_at":"2026-07-05T10:20:38.393547+00:00"}