{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:STT55N72ETHKS2URH5BEV45X5I","short_pith_number":"pith:STT55N72","schema_version":"1.0","canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","source":{"kind":"arxiv","id":"2408.04667","version":5},"attestation_state":"computed","paper":{"title":"Non-Determinism of \"Deterministic\" LLM Settings","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.SE"],"primary_cat":"cs.CL","authors_text":"Adam Sloan, Alexa Chittams, Berk Atil, Breck Baldwin, Evan Radcliffe, Ferhan Ture, Guru Rajan Rajagopal, Lisheng Fu, Lixinyu Xu, Rebecca J. Passonneau, Sarp Aykent, Tomasz Tudrej, Zhe Wu","submitted_at":"2024-08-06T16:43:35Z","abstract_excerpt":"LLM (large language model) practitioners commonly notice that outputs can vary for the same inputs under settings expected to be deterministic. Yet the questions of how pervasive this is, and with what impact on results, have not to our knowledge been systematically investigated. We investigate non-determinism in five LLMs configured to be deterministic when applied to eight common tasks in across 10 runs, in both zero-shot and few-shot settings. We see accuracy variations up to 15% across naturally occurring runs with a gap of best possible performance to worst possible performance up to 70%."},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.04667","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","cross_cats_sorted":["cs.AI","cs.LG","cs.SE"],"title_canon_sha256":"86bda973b208d281e2da204da74e7fa6a4ec39ed5f1eeffc34588aad8fb4cf50","abstract_canon_sha256":"0d6e3b5effa3aa5d479770b752184e0d92e79784576abe50af0895e20729fdf3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:43:25.246618Z","signature_b64":"cEsYGg4+zy8ZL6ixGsCZPQ755VBlhv9AYb2z2Q1+D+ieWxzeGVok2oGjIPpDzxZWxbbRt5VzXQTJR+j2lKVODQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","last_reissued_at":"2026-07-05T10:43:25.246129Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:43:25.246129Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Non-Determinism of \"Deterministic\" LLM Settings","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.SE"],"primary_cat":"cs.CL","authors_text":"Adam Sloan, Alexa Chittams, Berk Atil, Breck Baldwin, Evan Radcliffe, Ferhan Ture, Guru Rajan Rajagopal, Lisheng Fu, Lixinyu Xu, Rebecca J. Passonneau, Sarp Aykent, Tomasz Tudrej, Zhe Wu","submitted_at":"2024-08-06T16:43:35Z","abstract_excerpt":"LLM (large language model) practitioners commonly notice that outputs can vary for the same inputs under settings expected to be deterministic. Yet the questions of how pervasive this is, and with what impact on results, have not to our knowledge been systematically investigated. We investigate non-determinism in five LLMs configured to be deterministic when applied to eight common tasks in across 10 runs, in both zero-shot and few-shot settings. We see accuracy variations up to 15% across naturally occurring runs with a gap of best possible performance to worst possible performance up to 70%."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.04667","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.04667/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.04667","created_at":"2026-07-05T10:43:25.246187+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.04667v5","created_at":"2026-07-05T10:43:25.246187+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.04667","created_at":"2026-07-05T10:43:25.246187+00:00"},{"alias_kind":"pith_short_12","alias_value":"STT55N72ETHK","created_at":"2026-07-05T10:43:25.246187+00:00"},{"alias_kind":"pith_short_16","alias_value":"STT55N72ETHKS2UR","created_at":"2026-07-05T10:43:25.246187+00:00"},{"alias_kind":"pith_short_8","alias_value":"STT55N72","created_at":"2026-07-05T10:43:25.246187+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":23,"internal_anchor_count":3,"sample":[{"citing_arxiv_id":"2607.07766","citing_title":"Alignment Plausibility: A New Standard for Assuring AI in Healthcare","ref_index":55,"is_internal_anchor":true},{"citing_arxiv_id":"2607.07985","citing_title":"A Reliability Assessment of LALM Audio Judges for Full-Duplex Voice Agents","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2607.06229","citing_title":"Spider 2.0-AIFunc: Extending Real-World Text-to-SQL to AI-Native SQL Workflows","ref_index":4,"is_internal_anchor":true},{"citing_arxiv_id":"2605.02096","citing_title":"Foundation Models as Oracles for Refactoring Correctness Detection","ref_index":35,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03142","citing_title":"Disentangling Visual and Factual Correctness in LVLMs' Visualization Literacy","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03330","citing_title":"FLIPS: Instance-Fingerprinting for LLMs via Pseudo-random Sequences","ref_index":46,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27440","citing_title":"Paraphrase Brittleness in Production Retrieval-Augmented Commercial Recommendation: Reproducibility Below the Rerun-Stability Baseline","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25101","citing_title":"Multi-Agent Specification-based Metamorphic Testing of FMU-Based Simulations","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27763","citing_title":"A Paired Testing Protocol for Batch-Conditioned Refusal Robustness in LLM Serving","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28321","citing_title":"Multi-Agent LLM-based Metamorphic Testing for REST APIs","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2505.15134","citing_title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2511.20284","citing_title":"Can LLMs Make (Personalized) Access Control Decisions?","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16310","citing_title":"RAG-DIVE: A Dynamic Approach for Multi-Turn Dialogue Evaluation in Retrieval-Augmented Generation","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2602.17469","citing_title":"Cross-Lingual Sentiment Misalignment: Auditing Multilingual Language Models for Inversion Risk, Dialectal Representation, and Affective Stability","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2603.08819","citing_title":"Beyond Relevance: On the Relationship Between Retrieval and RAG Information Coverage","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27006","citing_title":"Beyond Accuracy: LLM Variability in Evidence Screening for Software Engineering SLRs","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2604.22176","citing_title":"FixV2W: Correcting Invalid CVE-CWE Mappings with Knowledge Graph Embeddings","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2604.22411","citing_title":"Introducing Background Temperature to Characterise Hidden Randomness in Large Language Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19598","citing_title":"Cross-Model Consistency of AI-Generated Exercise Prescriptions: A Repeated Generation Study Across Three Large Language Models","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09998","citing_title":"Like a Hammer, It Can Build, It Can Break: Large Language Model Uses, Perceptions, and Adoption in Cybersecurity Operations on Reddit","ref_index":68,"is_internal_anchor":false},{"citing_arxiv_id":"2604.05150","citing_title":"Compiled AI: Deterministic Code Generation for LLM-Based Workflow Automation","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18775","citing_title":"An Empirical Study of Multi-Generation Sampling for Jailbreak Detection in Large Language Models","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02096","citing_title":"Foundation Models as Oracles for Refactoring Correctness Detection","ref_index":35,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I","json":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I.json","graph_json":"https://pith.science/api/pith-number/STT55N72ETHKS2URH5BEV45X5I/graph.json","events_json":"https://pith.science/api/pith-number/STT55N72ETHKS2URH5BEV45X5I/events.json","paper":"https://pith.science/paper/STT55N72"},"agent_actions":{"view_html":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I","download_json":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I.json","view_paper":"https://pith.science/paper/STT55N72","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.04667&json=true","fetch_graph":"https://pith.science/api/pith-number/STT55N72ETHKS2URH5BEV45X5I/graph.json","fetch_events":"https://pith.science/api/pith-number/STT55N72ETHKS2URH5BEV45X5I/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/action/timestamp_anchor","attest_storage":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/action/storage_attestation","attest_author":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/action/author_attestation","sign_citation":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/action/citation_signature","submit_replication":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/action/replication_record"}},"created_at":"2026-07-05T10:43:25.246187+00:00","updated_at":"2026-07-05T10:43:25.246187+00:00"}