{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:4XPFWIKY77C5MD5N7F4GKURWBE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8605c40c55e8a3569fa064e5c937180c5a2dce2abbc946926f16a231a5d427a3","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-03T17:15:17Z","title_canon_sha256":"07f0a7a246295f78b8004bc4b1a65ba5518f4a5d5c70aebe0ce40818ffec4e0d"},"schema_version":"1.0","source":{"id":"2503.01747","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.01747","created_at":"2026-07-05T11:11:21Z"},{"alias_kind":"arxiv_version","alias_value":"2503.01747v3","created_at":"2026-07-05T11:11:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.01747","created_at":"2026-07-05T11:11:21Z"},{"alias_kind":"pith_short_12","alias_value":"4XPFWIKY77C5","created_at":"2026-07-05T11:11:21Z"},{"alias_kind":"pith_short_16","alias_value":"4XPFWIKY77C5MD5N","created_at":"2026-07-05T11:11:21Z"},{"alias_kind":"pith_short_8","alias_value":"4XPFWIKY","created_at":"2026-07-05T11:11:21Z"}],"graph_snapshots":[{"event_id":"sha256:d78898dac550b1bb6f51b201da3a513cac39e58373d24158007b06d6f5dcf99e","target":"graph","created_at":"2026-07-05T11:11:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.01747/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Rigorous statistical evaluations of large language models (LLMs), including valid error bars and significance testing, are essential for meaningful and reliable performance assessment. Currently, when such statistical measures are reported, they typically rely on the Central Limit Theorem (CLT). In this position paper, we argue that while CLT-based methods for uncertainty quantification are appropriate when benchmarks consist of thousands of examples, they fail to provide adequate uncertainty estimates for LLM evaluations that rely on smaller, highly specialized benchmarks. In these small-data","authors_text":"Desi R. Ivanova, Laurence Aitchison, Sam Bowyer","cross_cats":["cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-03T17:15:17Z","title":"Position: Don't Use the CLT in LLM Evals With Fewer Than a Few Hundred Datapoints"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.01747","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cf79d39b0fa496b3930910026b0ca20ad3ad0f5dec8f7534d8ec0aec47cd38eb","target":"record","created_at":"2026-07-05T11:11:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8605c40c55e8a3569fa064e5c937180c5a2dce2abbc946926f16a231a5d427a3","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-03T17:15:17Z","title_canon_sha256":"07f0a7a246295f78b8004bc4b1a65ba5518f4a5d5c70aebe0ce40818ffec4e0d"},"schema_version":"1.0","source":{"id":"2503.01747","kind":"arxiv","version":3}},"canonical_sha256":"e5de5b2158ffc5d60fadf978655236090768532d16be9e382be9827568f16653","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e5de5b2158ffc5d60fadf978655236090768532d16be9e382be9827568f16653","first_computed_at":"2026-07-05T11:11:21.824581Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:11:21.824581Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"OVuYy43971nFwMW/egDBQaMpyan6kpz7LeSzA62VuDrNmgGDW7rzrIj5pwTCqcso7LQvn9ewNR5UNsLECxDADA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:11:21.825036Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.01747","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cf79d39b0fa496b3930910026b0ca20ad3ad0f5dec8f7534d8ec0aec47cd38eb","sha256:d78898dac550b1bb6f51b201da3a513cac39e58373d24158007b06d6f5dcf99e"],"state_sha256":"bb152f19f8ac217a38bfe53eb6b6f461f4caa642dcfd9fd878b053716702cbd0"}