{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:H2PM6DFQMM5O6CJYSJMJIIQ5HA","short_pith_number":"pith:H2PM6DFQ","canonical_record":{"source":{"id":"2409.03563","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-05T14:19:45Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"0065ba5c1383bbcd7090b5f56d60d8115962cfd3f2d6f5c7df18767c33bc997d","abstract_canon_sha256":"c95143b07f94c9a89f66b85eb18f5f1629b33ba31fc861a9c0b2d34cbb7d6f54"},"schema_version":"1.0"},"canonical_sha256":"3e9ecf0cb0633aef0938925894221d3808db1e1b0f48ebc0fb23afa1b42934c3","source":{"kind":"arxiv","id":"2409.03563","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.03563","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"arxiv_version","alias_value":"2409.03563v1","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.03563","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_12","alias_value":"H2PM6DFQMM5O","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_16","alias_value":"H2PM6DFQMM5O6CJY","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_8","alias_value":"H2PM6DFQ","created_at":"2026-07-05T09:03:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:H2PM6DFQMM5O6CJYSJMJIIQ5HA","target":"record","payload":{"canonical_record":{"source":{"id":"2409.03563","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-05T14:19:45Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"0065ba5c1383bbcd7090b5f56d60d8115962cfd3f2d6f5c7df18767c33bc997d","abstract_canon_sha256":"c95143b07f94c9a89f66b85eb18f5f1629b33ba31fc861a9c0b2d34cbb7d6f54"},"schema_version":"1.0"},"canonical_sha256":"3e9ecf0cb0633aef0938925894221d3808db1e1b0f48ebc0fb23afa1b42934c3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:03:33.254011Z","signature_b64":"aMDipgwH3Vy8z7jOmwuTdkvzahLndXKYw2Q0v7qHPfPTfiAtvJ3OvFztdhmlzpezWGsQLSfJWMbgWCwCSzhJBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3e9ecf0cb0633aef0938925894221d3808db1e1b0f48ebc0fb23afa1b42934c3","last_reissued_at":"2026-07-05T09:03:33.253554Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:03:33.253554Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.03563","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:03:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EiBWbbqqtppZ67m0dgNUqkgi6KMblvmRMI7aFF0vHOx9/wH/UU6ukrqixKkoCLEJ/Y7akdwC6yXIfOZIcmYJAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:15:29.395657Z"},"content_sha256":"3355949659f62427f028995f869a8b694dd40426fd3923880a9cc9809339c90f","schema_version":"1.0","event_id":"sha256:3355949659f62427f028995f869a8b694dd40426fd3923880a9cc9809339c90f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:H2PM6DFQMM5O6CJYSJMJIIQ5HA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"100 instances is all you need: predicting the success of a new LLM on unseen data by testing on a few instances","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Jos\\'e Hern\\'andez-Orallo, Lorenzo Pacchiardi, Lucy G. Cheke","submitted_at":"2024-09-05T14:19:45Z","abstract_excerpt":"Predicting the performance of LLMs on individual task instances is essential to ensure their reliability in high-stakes applications. To do so, a possibility is to evaluate the considered LLM on a set of task instances and train an assessor to predict its performance based on features of the instances. However, this approach requires evaluating each new LLM on a sufficiently large set of task instances to train an assessor specific to it. In this work, we leverage the evaluation results of previously tested LLMs to reduce the number of evaluations required to predict the performance of a new L"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.03563","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.03563/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:03:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PxFFBbKswNY23pzGEFRn7iFKZhEqTSgD+jmVnETvdmboai5xmLZy8NynMOWIYR8qzjF7WlqMufdTqhDyjarGBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:15:29.396181Z"},"content_sha256":"835bda04264ef6c0e9b1cd619777d0fdbbbba8dc463016e8053af9b77f1f4e72","schema_version":"1.0","event_id":"sha256:835bda04264ef6c0e9b1cd619777d0fdbbbba8dc463016e8053af9b77f1f4e72"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/bundle.json","state_url":"https://pith.science/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T17:15:29Z","links":{"resolver":"https://pith.science/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA","bundle":"https://pith.science/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/bundle.json","state":"https://pith.science/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/H2PM6DFQMM5O6CJYSJMJIIQ5HA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:H2PM6DFQMM5O6CJYSJMJIIQ5HA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c95143b07f94c9a89f66b85eb18f5f1629b33ba31fc861a9c0b2d34cbb7d6f54","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-05T14:19:45Z","title_canon_sha256":"0065ba5c1383bbcd7090b5f56d60d8115962cfd3f2d6f5c7df18767c33bc997d"},"schema_version":"1.0","source":{"id":"2409.03563","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.03563","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"arxiv_version","alias_value":"2409.03563v1","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.03563","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_12","alias_value":"H2PM6DFQMM5O","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_16","alias_value":"H2PM6DFQMM5O6CJY","created_at":"2026-07-05T09:03:33Z"},{"alias_kind":"pith_short_8","alias_value":"H2PM6DFQ","created_at":"2026-07-05T09:03:33Z"}],"graph_snapshots":[{"event_id":"sha256:835bda04264ef6c0e9b1cd619777d0fdbbbba8dc463016e8053af9b77f1f4e72","target":"graph","created_at":"2026-07-05T09:03:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.03563/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Predicting the performance of LLMs on individual task instances is essential to ensure their reliability in high-stakes applications. To do so, a possibility is to evaluate the considered LLM on a set of task instances and train an assessor to predict its performance based on features of the instances. However, this approach requires evaluating each new LLM on a sufficiently large set of task instances to train an assessor specific to it. In this work, we leverage the evaluation results of previously tested LLMs to reduce the number of evaluations required to predict the performance of a new L","authors_text":"Jos\\'e Hern\\'andez-Orallo, Lorenzo Pacchiardi, Lucy G. Cheke","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-05T14:19:45Z","title":"100 instances is all you need: predicting the success of a new LLM on unseen data by testing on a few instances"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.03563","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3355949659f62427f028995f869a8b694dd40426fd3923880a9cc9809339c90f","target":"record","created_at":"2026-07-05T09:03:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c95143b07f94c9a89f66b85eb18f5f1629b33ba31fc861a9c0b2d34cbb7d6f54","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-05T14:19:45Z","title_canon_sha256":"0065ba5c1383bbcd7090b5f56d60d8115962cfd3f2d6f5c7df18767c33bc997d"},"schema_version":"1.0","source":{"id":"2409.03563","kind":"arxiv","version":1}},"canonical_sha256":"3e9ecf0cb0633aef0938925894221d3808db1e1b0f48ebc0fb23afa1b42934c3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3e9ecf0cb0633aef0938925894221d3808db1e1b0f48ebc0fb23afa1b42934c3","first_computed_at":"2026-07-05T09:03:33.253554Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:03:33.253554Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"aMDipgwH3Vy8z7jOmwuTdkvzahLndXKYw2Q0v7qHPfPTfiAtvJ3OvFztdhmlzpezWGsQLSfJWMbgWCwCSzhJBA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:03:33.254011Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.03563","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3355949659f62427f028995f869a8b694dd40426fd3923880a9cc9809339c90f","sha256:835bda04264ef6c0e9b1cd619777d0fdbbbba8dc463016e8053af9b77f1f4e72"],"state_sha256":"1aaed56b1ec2af592567be6492186bc4a9ba0a5c1cc9c94148a1c87641754f77"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UqegJTS3XDUttq2NLSx4Xc/Cv9yIIWtoy/fhVneDJHwC2sxthnweE3FXM45y26o1zPMORP7cGoBAegFPVUO0Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T17:15:29.399978Z","bundle_sha256":"b3ea836426952beb587b8699118a1098506cde92238c65e423fefc712b5b20bf"}}