{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:J6U2L5ULUPBBPV4JYCNZK4D7VT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21"},"schema_version":"1.0","source":{"id":"2405.17202","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17202v3","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_12","alias_value":"J6U2L5ULUPBB","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_16","alias_value":"J6U2L5ULUPBBPV4J","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_8","alias_value":"J6U2L5UL","created_at":"2026-07-05T09:28:50Z"}],"graph_snapshots":[{"event_id":"sha256:9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b","target":"graph","created_at":"2026-07-05T09:28:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.17202/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most popular benchmarks for comparing LLMs rely on a limited set of prompt templates, which may not fully capture the LLMs' abilities and can affect the reproducibility of results on leaderboards. Many recent works empirically verify prompt sensitivity and advocate for changes in LLM evaluation. In this paper, we consider the problem of estimating the performance distribution across many prompt variants instead of finding a single prompt to evaluate with. We introduce PromptEval, a method for estimating performance across a large set of prompts borrowing strength across prompts and examples to","authors_text":"Allysson Flavio Melo de Oliveira, Felipe Maia Polo, Leshem Choshen, Lucas Weber, Mikhail Yurochkin, M\\'irian Silva, Onkar Bhardwaj, Ronald Xu, Yuekai Sun","cross_cats":["cs.AI","cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title":"Efficient multi-prompt evaluation of LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17202","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51","target":"record","created_at":"2026-07-05T09:28:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21"},"schema_version":"1.0","source":{"id":"2405.17202","kind":"arxiv","version":3}},"canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","first_computed_at":"2026-07-05T09:28:50.310392Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:28:50.310392Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X/U4Eg0PhdCzZwQlZvBkGGzisPp2DuaTgttIzKLCbN/3LO9bDFZHAhGLjiCLYx+FBHxmOvOC6IfkOPi0pXH2Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:28:50.310907Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.17202","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51","sha256:9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b"],"state_sha256":"a0aef9d32377dfe7b9362f637a53ecadb1967cdd7c2966f3b5f45fe74647ef07"}