{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:RZ36HTG6DZ37N2NPSIZP4ZT2T2","short_pith_number":"pith:RZ36HTG6","canonical_record":{"source":{"id":"2305.14802","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T06:55:09Z","cross_cats_sorted":[],"title_canon_sha256":"42df1a60a02fd5d6912841f88e4acc59b9211fea4a0f8d61440f07565314d62f","abstract_canon_sha256":"0a46d287a840d2b5da0f8e43f5bf3e5b3f3bda677912063132d7466e0d4ed1e8"},"schema_version":"1.0"},"canonical_sha256":"8e77e3ccde1e77f6e9af9232fe667a9e9fe0a743f37d16f02396fd45d65bd5dc","source":{"kind":"arxiv","id":"2305.14802","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14802","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14802v2","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14802","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_12","alias_value":"RZ36HTG6DZ37","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_16","alias_value":"RZ36HTG6DZ37N2NP","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_8","alias_value":"RZ36HTG6","created_at":"2026-07-05T07:05:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:RZ36HTG6DZ37N2NPSIZP4ZT2T2","target":"record","payload":{"canonical_record":{"source":{"id":"2305.14802","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T06:55:09Z","cross_cats_sorted":[],"title_canon_sha256":"42df1a60a02fd5d6912841f88e4acc59b9211fea4a0f8d61440f07565314d62f","abstract_canon_sha256":"0a46d287a840d2b5da0f8e43f5bf3e5b3f3bda677912063132d7466e0d4ed1e8"},"schema_version":"1.0"},"canonical_sha256":"8e77e3ccde1e77f6e9af9232fe667a9e9fe0a743f37d16f02396fd45d65bd5dc","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:05:09.025531Z","signature_b64":"zMWw58rorlFRBcA4lo0RW/PYGqbk9oW+/tTZqCUjZLhZDOMFspndPoEWSJ93Ab9DJ2c/Os2tW6K7WakE4AVIAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8e77e3ccde1e77f6e9af9232fe667a9e9fe0a743f37d16f02396fd45d65bd5dc","last_reissued_at":"2026-07-05T07:05:09.025037Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:05:09.025037Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.14802","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:05:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"paoHGVzIXJPwKnr7+ceBDjmraZLLhO77soMl56+rtJYjM1G8XicmX9zkEvf3JAGVnh4Ib4V1PY2sE6cM5NO7Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:21:22.053960Z"},"content_sha256":"048bd897f200fc7afb1aa069210d52800a9f46e08a93f4b2fc61ed8b4cabfd92","schema_version":"1.0","event_id":"sha256:048bd897f200fc7afb1aa069210d52800a9f46e08a93f4b2fc61ed8b4cabfd92"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:RZ36HTG6DZ37N2NPSIZP4ZT2T2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Estimating Large Language Model Capabilities without Labeled Test Data","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Albert Xu, Harvey Yiyun Fu, Qinyuan Ye, Robin Jia, Xiang Ren","submitted_at":"2023-05-24T06:55:09Z","abstract_excerpt":"Large Language Models (LLMs) have the impressive ability to perform in-context learning (ICL) from only a few examples, but the success of ICL varies widely from task to task. Thus, it is important to quickly determine whether ICL is applicable to a new task, but directly evaluating ICL accuracy can be expensive in situations where test data is expensive to annotate -- the exact situations where ICL is most appealing. In this paper, we propose the task of ICL accuracy estimation, in which we predict the accuracy of an LLM when doing in-context learning on a new task given only unlabeled test d"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14802","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.14802/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:05:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LSWsEaTExbW+sahC92AlttsdUGGSpicjF35xe1+bsXW7o3o1DttOYqCkRq2vHW8NeufTSPXeqPt9D4ZkXKoPDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:21:22.054352Z"},"content_sha256":"e4702fbd9c93e6fd889a9d970c3e5c128f1e8b6e68aa8e73e6d947ab642de3dc","schema_version":"1.0","event_id":"sha256:e4702fbd9c93e6fd889a9d970c3e5c128f1e8b6e68aa8e73e6d947ab642de3dc"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/bundle.json","state_url":"https://pith.science/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T19:21:22Z","links":{"resolver":"https://pith.science/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2","bundle":"https://pith.science/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/bundle.json","state":"https://pith.science/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RZ36HTG6DZ37N2NPSIZP4ZT2T2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:RZ36HTG6DZ37N2NPSIZP4ZT2T2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0a46d287a840d2b5da0f8e43f5bf3e5b3f3bda677912063132d7466e0d4ed1e8","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T06:55:09Z","title_canon_sha256":"42df1a60a02fd5d6912841f88e4acc59b9211fea4a0f8d61440f07565314d62f"},"schema_version":"1.0","source":{"id":"2305.14802","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.14802","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"arxiv_version","alias_value":"2305.14802v2","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.14802","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_12","alias_value":"RZ36HTG6DZ37","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_16","alias_value":"RZ36HTG6DZ37N2NP","created_at":"2026-07-05T07:05:09Z"},{"alias_kind":"pith_short_8","alias_value":"RZ36HTG6","created_at":"2026-07-05T07:05:09Z"}],"graph_snapshots":[{"event_id":"sha256:e4702fbd9c93e6fd889a9d970c3e5c128f1e8b6e68aa8e73e6d947ab642de3dc","target":"graph","created_at":"2026-07-05T07:05:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.14802/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have the impressive ability to perform in-context learning (ICL) from only a few examples, but the success of ICL varies widely from task to task. Thus, it is important to quickly determine whether ICL is applicable to a new task, but directly evaluating ICL accuracy can be expensive in situations where test data is expensive to annotate -- the exact situations where ICL is most appealing. In this paper, we propose the task of ICL accuracy estimation, in which we predict the accuracy of an LLM when doing in-context learning on a new task given only unlabeled test d","authors_text":"Albert Xu, Harvey Yiyun Fu, Qinyuan Ye, Robin Jia, Xiang Ren","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T06:55:09Z","title":"Estimating Large Language Model Capabilities without Labeled Test Data"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.14802","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:048bd897f200fc7afb1aa069210d52800a9f46e08a93f4b2fc61ed8b4cabfd92","target":"record","created_at":"2026-07-05T07:05:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0a46d287a840d2b5da0f8e43f5bf3e5b3f3bda677912063132d7466e0d4ed1e8","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-24T06:55:09Z","title_canon_sha256":"42df1a60a02fd5d6912841f88e4acc59b9211fea4a0f8d61440f07565314d62f"},"schema_version":"1.0","source":{"id":"2305.14802","kind":"arxiv","version":2}},"canonical_sha256":"8e77e3ccde1e77f6e9af9232fe667a9e9fe0a743f37d16f02396fd45d65bd5dc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8e77e3ccde1e77f6e9af9232fe667a9e9fe0a743f37d16f02396fd45d65bd5dc","first_computed_at":"2026-07-05T07:05:09.025037Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:05:09.025037Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zMWw58rorlFRBcA4lo0RW/PYGqbk9oW+/tTZqCUjZLhZDOMFspndPoEWSJ93Ab9DJ2c/Os2tW6K7WakE4AVIAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T07:05:09.025531Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.14802","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:048bd897f200fc7afb1aa069210d52800a9f46e08a93f4b2fc61ed8b4cabfd92","sha256:e4702fbd9c93e6fd889a9d970c3e5c128f1e8b6e68aa8e73e6d947ab642de3dc"],"state_sha256":"22096e561f68203d37a8d5f1719b704052425986eea97e860d94b7a244f40acb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eU8ONMXlQcZYQIbybBafwqMxeOl9SheG3H4crq27nw/D+rlDOTvmgZ//V/ZS+i8sr0VQXb1NXmTGgWuN7/TtCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T19:21:22.056351Z","bundle_sha256":"86455fa9cbc47310e78cfea06ae1328e738f4dc6bbb2b51ba4a4f32b84869775"}}