{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:J6U2L5ULUPBBPV4JYCNZK4D7VT","short_pith_number":"pith:J6U2L5UL","canonical_record":{"source":{"id":"2405.17202","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21","abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8"},"schema_version":"1.0"},"canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","source":{"kind":"arxiv","id":"2405.17202","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17202v3","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_12","alias_value":"J6U2L5ULUPBB","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_16","alias_value":"J6U2L5ULUPBBPV4J","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_8","alias_value":"J6U2L5UL","created_at":"2026-07-05T09:28:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:J6U2L5ULUPBBPV4JYCNZK4D7VT","target":"record","payload":{"canonical_record":{"source":{"id":"2405.17202","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21","abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8"},"schema_version":"1.0"},"canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:28:50.310907Z","signature_b64":"X/U4Eg0PhdCzZwQlZvBkGGzisPp2DuaTgttIzKLCbN/3LO9bDFZHAhGLjiCLYx+FBHxmOvOC6IfkOPi0pXH2Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","last_reissued_at":"2026-07-05T09:28:50.310392Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:28:50.310392Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.17202","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:28:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"djVn9YUOJJp/YGpt4+5htSZRHUpRwLTuI4iDdRcJ+7OOB7c3k6hNNL7W8J2qtSa/PXCsdNekJzVNkBh7lyKuCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:16:43.106508Z"},"content_sha256":"6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51","schema_version":"1.0","event_id":"sha256:6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:J6U2L5ULUPBBPV4JYCNZK4D7VT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Efficient multi-prompt evaluation of LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","stat.ML"],"primary_cat":"cs.CL","authors_text":"Allysson Flavio Melo de Oliveira, Felipe Maia Polo, Leshem Choshen, Lucas Weber, Mikhail Yurochkin, M\\'irian Silva, Onkar Bhardwaj, Ronald Xu, Yuekai Sun","submitted_at":"2024-05-27T14:24:47Z","abstract_excerpt":"Most popular benchmarks for comparing LLMs rely on a limited set of prompt templates, which may not fully capture the LLMs' abilities and can affect the reproducibility of results on leaderboards. Many recent works empirically verify prompt sensitivity and advocate for changes in LLM evaluation. In this paper, we consider the problem of estimating the performance distribution across many prompt variants instead of finding a single prompt to evaluate with. We introduce PromptEval, a method for estimating performance across a large set of prompts borrowing strength across prompts and examples to"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17202","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.17202/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:28:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ggfmu+7KJn5O0noO75L0LB/Jim0tK1gBux3aqlJwgCGf6VRpKfwCkDefiCD3RESYPleBDgocu/HK1O8db82aCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:16:43.106989Z"},"content_sha256":"9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b","schema_version":"1.0","event_id":"sha256:9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/bundle.json","state_url":"https://pith.science/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T04:16:43Z","links":{"resolver":"https://pith.science/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT","bundle":"https://pith.science/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/bundle.json","state":"https://pith.science/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/J6U2L5ULUPBBPV4JYCNZK4D7VT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:J6U2L5ULUPBBPV4JYCNZK4D7VT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21"},"schema_version":"1.0","source":{"id":"2405.17202","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17202v3","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17202","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_12","alias_value":"J6U2L5ULUPBB","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_16","alias_value":"J6U2L5ULUPBBPV4J","created_at":"2026-07-05T09:28:50Z"},{"alias_kind":"pith_short_8","alias_value":"J6U2L5UL","created_at":"2026-07-05T09:28:50Z"}],"graph_snapshots":[{"event_id":"sha256:9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b","target":"graph","created_at":"2026-07-05T09:28:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.17202/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most popular benchmarks for comparing LLMs rely on a limited set of prompt templates, which may not fully capture the LLMs' abilities and can affect the reproducibility of results on leaderboards. Many recent works empirically verify prompt sensitivity and advocate for changes in LLM evaluation. In this paper, we consider the problem of estimating the performance distribution across many prompt variants instead of finding a single prompt to evaluate with. We introduce PromptEval, a method for estimating performance across a large set of prompts borrowing strength across prompts and examples to","authors_text":"Allysson Flavio Melo de Oliveira, Felipe Maia Polo, Leshem Choshen, Lucas Weber, Mikhail Yurochkin, M\\'irian Silva, Onkar Bhardwaj, Ronald Xu, Yuekai Sun","cross_cats":["cs.AI","cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title":"Efficient multi-prompt evaluation of LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17202","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51","target":"record","created_at":"2026-07-05T09:28:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"709c87a46fc77f46a31930004a156133ff5ae5b8f513ec0b8255fe5f750152d8","cross_cats_sorted":["cs.AI","cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T14:24:47Z","title_canon_sha256":"f1f5ef1e060acb2448e04569837586105ee37fa0f8f67b3fca80df37fd35fc21"},"schema_version":"1.0","source":{"id":"2405.17202","kind":"arxiv","version":3}},"canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4fa9a5f68ba3c217d789c09b95707facf763ad331e4ab85d86f9bc3e0de17395","first_computed_at":"2026-07-05T09:28:50.310392Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:28:50.310392Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X/U4Eg0PhdCzZwQlZvBkGGzisPp2DuaTgttIzKLCbN/3LO9bDFZHAhGLjiCLYx+FBHxmOvOC6IfkOPi0pXH2Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:28:50.310907Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.17202","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6bcf15146f2f552b13cbf6460a4ced49c64de44483fcc74e97ed4823456e3e51","sha256:9c93b581f28ac8275cce86a52d9b19ff33c05d4fa68366d0f4dc986c86444d2b"],"state_sha256":"a0aef9d32377dfe7b9362f637a53ecadb1967cdd7c2966f3b5f45fe74647ef07"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vfZxZqvmb8ICUL7qYv4+Mvm1N03FnF7e5uF2vtn9/f+lT0IArlvYW4ldKSfnRpOpcueSsEB6hFKtr5gAB/lkCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T04:16:43.109360Z","bundle_sha256":"d3f8ad004af426c0d9ce975601729215c9b3fd7a988eed4626d1c1ae84c96262"}}