{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:WCX7CJS6QADAF7MGEI372ROJPC","short_pith_number":"pith:WCX7CJS6","schema_version":"1.0","canonical_sha256":"b0aff1265e800602fd862237fd45c978bef519fc80d259029d97d0a568221263","source":{"kind":"arxiv","id":"2410.03492","version":2},"attestation_state":"computed","paper":{"title":"Towards Reproducible LLM Evaluation: Quantifying Uncertainty in LLM Benchmark Scores","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anthony G. Cohn, Jon Barry, Robert E. Blackwell","submitted_at":"2024-10-04T15:04:28Z","abstract_excerpt":"Large language models (LLMs) are stochastic, and not all models give deterministic answers, even when setting temperature to zero with a fixed random seed. However, few benchmark studies attempt to quantify uncertainty, partly due to the time and cost of repeated experiments. We use benchmarks designed for testing LLMs' capacity to reason about cardinal directions to explore the impact of experimental repeats on mean score and prediction interval. We suggest a simple method for cost-effectively quantifying the uncertainty of a benchmark score and make recommendations concerning reproducible LL"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.03492","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-04T15:04:28Z","cross_cats_sorted":[],"title_canon_sha256":"9d374d3fbaf24faa8a899ef1c2c05be0493a6a7a7dcc4ab287501fcaa81f36ba","abstract_canon_sha256":"7bf023c6547c0abae2345628ae4ac61f3d76c72ed7ac45341f752dc2645f7aa4"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:28:11.418662Z","signature_b64":"YJp8nS8v3xqtdo4eASQFmcd2aMnp1+zS3rqntSO0AXjKJa3oLrUvjJZfoVCkACj70abBhFTZUKLXDiqGZBwSBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b0aff1265e800602fd862237fd45c978bef519fc80d259029d97d0a568221263","last_reissued_at":"2026-07-05T11:28:11.418156Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:28:11.418156Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Towards Reproducible LLM Evaluation: Quantifying Uncertainty in LLM Benchmark Scores","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anthony G. Cohn, Jon Barry, Robert E. Blackwell","submitted_at":"2024-10-04T15:04:28Z","abstract_excerpt":"Large language models (LLMs) are stochastic, and not all models give deterministic answers, even when setting temperature to zero with a fixed random seed. However, few benchmark studies attempt to quantify uncertainty, partly due to the time and cost of repeated experiments. We use benchmarks designed for testing LLMs' capacity to reason about cardinal directions to explore the impact of experimental repeats on mean score and prediction interval. We suggest a simple method for cost-effectively quantifying the uncertainty of a benchmark score and make recommendations concerning reproducible LL"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.03492","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.03492/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.03492","created_at":"2026-07-05T11:28:11.418223+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.03492v2","created_at":"2026-07-05T11:28:11.418223+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.03492","created_at":"2026-07-05T11:28:11.418223+00:00"},{"alias_kind":"pith_short_12","alias_value":"WCX7CJS6QADA","created_at":"2026-07-05T11:28:11.418223+00:00"},{"alias_kind":"pith_short_16","alias_value":"WCX7CJS6QADAF7MG","created_at":"2026-07-05T11:28:11.418223+00:00"},{"alias_kind":"pith_short_8","alias_value":"WCX7CJS6","created_at":"2026-07-05T11:28:11.418223+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.19537","citing_title":"The Silent Hyperparameter: Quantifying the Impact of Inference Backends on LLM Reproducibility","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18380","citing_title":"QSTRBench: a New Benchmark to Evaluate the Ability of Language Models to Reason with Qualitative Spatial and Temporal Calculi","ref_index":67,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19537","citing_title":"The Silent Hyperparameter: Quantifying the Impact of Inference Backends on LLM Reproducibility","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11261","citing_title":"Inspectable AI for Science: A Research Object Approach to Generative AI Governance","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02091","citing_title":"How Compliant Are GitHub Actions Workflows? A Checklist-Based Study with LLM-Assisted Auditing","ref_index":5,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC","json":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC.json","graph_json":"https://pith.science/api/pith-number/WCX7CJS6QADAF7MGEI372ROJPC/graph.json","events_json":"https://pith.science/api/pith-number/WCX7CJS6QADAF7MGEI372ROJPC/events.json","paper":"https://pith.science/paper/WCX7CJS6"},"agent_actions":{"view_html":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC","download_json":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC.json","view_paper":"https://pith.science/paper/WCX7CJS6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.03492&json=true","fetch_graph":"https://pith.science/api/pith-number/WCX7CJS6QADAF7MGEI372ROJPC/graph.json","fetch_events":"https://pith.science/api/pith-number/WCX7CJS6QADAF7MGEI372ROJPC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC/action/storage_attestation","attest_author":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC/action/author_attestation","sign_citation":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC/action/citation_signature","submit_replication":"https://pith.science/pith/WCX7CJS6QADAF7MGEI372ROJPC/action/replication_record"}},"created_at":"2026-07-05T11:28:11.418223+00:00","updated_at":"2026-07-05T11:28:11.418223+00:00"}