{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:QJFVZJSL3MAFJLAP2A5PCRXF4Z","short_pith_number":"pith:QJFVZJSL","schema_version":"1.0","canonical_sha256":"824b5ca64bdb0054ac0fd03af146e5e645f3df353043df0bcd1903421b1d66fd","source":{"kind":"arxiv","id":"2411.00640","version":1},"attestation_state":"computed","paper":{"title":"Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"stat.AP","authors_text":"Evan Miller","submitted_at":"2024-11-01T14:57:16Z","abstract_excerpt":"Evaluations are critical for understanding the capabilities of large language models (LLMs). Fundamentally, evaluations are experiments; but the literature on evaluations has largely ignored the literature from other sciences on experiment analysis and planning. This article shows researchers with some training in statistics how to think about and analyze data from language model evaluations. Conceptualizing evaluation questions as having been drawn from an unseen super-population, we present formulas for analyzing evaluation data, measuring differences between two models, and planning an eval"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.00640","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2024-11-01T14:57:16Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"986f5ba6fd4cc287cda3aa767e08afe4355d452bf75f797bbb8058e0576b8611","abstract_canon_sha256":"dd4eda217022718702bbebbbf50a92212361e54fbfa9795a508374e4f4f31939"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:29:46.059457Z","signature_b64":"dAnAScsWYs0wJ/OhNl1XRAOCghbCN387SvqprAO2zzN28P7RCbs5zvlTHaG63iLAwpKTr/FgYcqNTfGR8742Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"824b5ca64bdb0054ac0fd03af146e5e645f3df353043df0bcd1903421b1d66fd","last_reissued_at":"2026-07-05T09:29:46.058980Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:29:46.058980Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"stat.AP","authors_text":"Evan Miller","submitted_at":"2024-11-01T14:57:16Z","abstract_excerpt":"Evaluations are critical for understanding the capabilities of large language models (LLMs). Fundamentally, evaluations are experiments; but the literature on evaluations has largely ignored the literature from other sciences on experiment analysis and planning. This article shows researchers with some training in statistics how to think about and analyze data from language model evaluations. Conceptualizing evaluation questions as having been drawn from an unseen super-population, we present formulas for analyzing evaluation data, measuring differences between two models, and planning an eval"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.00640","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.00640/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.00640","created_at":"2026-07-05T09:29:46.059038+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.00640v1","created_at":"2026-07-05T09:29:46.059038+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.00640","created_at":"2026-07-05T09:29:46.059038+00:00"},{"alias_kind":"pith_short_12","alias_value":"QJFVZJSL3MAF","created_at":"2026-07-05T09:29:46.059038+00:00"},{"alias_kind":"pith_short_16","alias_value":"QJFVZJSL3MAFJLAP","created_at":"2026-07-05T09:29:46.059038+00:00"},{"alias_kind":"pith_short_8","alias_value":"QJFVZJSL","created_at":"2026-07-05T09:29:46.059038+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":21,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.06411","citing_title":"RuBench: A Repository-Level Agentic Coding Benchmark with Natively Authored Russian Task Specifications","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2606.25487","citing_title":"How Reliable Is Your Jailbreak Judge? Calibration and Adversarial Robustness of Automated ASR Scoring","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.25487","citing_title":"How Reliable Is Your Jailbreak Judge? Calibration and Adversarial Robustness of Automated ASR Scoring","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2607.01245","citing_title":"Office Comprehension Benchmark","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2606.17426","citing_title":"Bounded Difference Concentration for Infinitely Exchangeable Sequences with Applications to AI Benchmark Uncertainty","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09809","citing_title":"Evaluation Cards: An Interpretive Layer for AI Evaluation Reporting","ref_index":75,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08679","citing_title":"Rank Intervals for Leaderboards: A Hierarchical Framework for Model Evaluation","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05029","citing_title":"Validity Threats for Foundation Model Research","ref_index":68,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30758","citing_title":"Pairwise Reference Alignment as a Model-Level Ordinal Observable","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30315","citing_title":"Resolution Diagnostics for Paired LLM Evaluation","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23362","citing_title":"Instance-Optimal Estimation with Multiple LLM Judges on a Budget","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2503.02574","citing_title":"LLM-Safety Evaluations Lack Robustness","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2509.19590","citing_title":"Position: AI Evaluations Should be Grounded on a Theory of Capability","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2511.01680","citing_title":"Making Interpretable Discoveries from Unstructured Data: A High-Dimensional Multiple Hypothesis Testing Approach","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2512.05929","citing_title":"LLM Harms: A Taxonomy and Discussion","ref_index":233,"is_internal_anchor":false},{"citing_arxiv_id":"2602.10144","citing_title":"When LLMs get significantly worse: A statistical approach to detect model degradations","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2603.04601","citing_title":"Vibe Code Bench: Evaluating AI Models on End-to-End Web Application Development","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2604.03472","citing_title":"Vocabulary Dropout for Curriculum Diversity in LLM Co-Evolution","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10575","citing_title":"Acceptance Cards:A Four-Diagnostic Standard for Safe Fine-Tuning Defense Claims","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06939","citing_title":"Bias and Uncertainty in LLM-as-a-Judge Estimation","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19822","citing_title":"Statistical Software Engineering with Tuned Variables","ref_index":18,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z","json":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z.json","graph_json":"https://pith.science/api/pith-number/QJFVZJSL3MAFJLAP2A5PCRXF4Z/graph.json","events_json":"https://pith.science/api/pith-number/QJFVZJSL3MAFJLAP2A5PCRXF4Z/events.json","paper":"https://pith.science/paper/QJFVZJSL"},"agent_actions":{"view_html":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z","download_json":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z.json","view_paper":"https://pith.science/paper/QJFVZJSL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.00640&json=true","fetch_graph":"https://pith.science/api/pith-number/QJFVZJSL3MAFJLAP2A5PCRXF4Z/graph.json","fetch_events":"https://pith.science/api/pith-number/QJFVZJSL3MAFJLAP2A5PCRXF4Z/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z/action/timestamp_anchor","attest_storage":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z/action/storage_attestation","attest_author":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z/action/author_attestation","sign_citation":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z/action/citation_signature","submit_replication":"https://pith.science/pith/QJFVZJSL3MAFJLAP2A5PCRXF4Z/action/replication_record"}},"created_at":"2026-07-05T09:29:46.059038+00:00","updated_at":"2026-07-05T09:29:46.059038+00:00"}