{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:STT55N72ETHKS2URH5BEV45X5I","short_pith_number":"pith:STT55N72","canonical_record":{"source":{"id":"2408.04667","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","cross_cats_sorted":["cs.AI","cs.LG","cs.SE"],"title_canon_sha256":"86bda973b208d281e2da204da74e7fa6a4ec39ed5f1eeffc34588aad8fb4cf50","abstract_canon_sha256":"0d6e3b5effa3aa5d479770b752184e0d92e79784576abe50af0895e20729fdf3"},"schema_version":"1.0"},"canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","source":{"kind":"arxiv","id":"2408.04667","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.04667","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"arxiv_version","alias_value":"2408.04667v5","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.04667","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_12","alias_value":"STT55N72ETHK","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_16","alias_value":"STT55N72ETHKS2UR","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_8","alias_value":"STT55N72","created_at":"2026-07-05T10:43:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:STT55N72ETHKS2URH5BEV45X5I","target":"record","payload":{"canonical_record":{"source":{"id":"2408.04667","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","cross_cats_sorted":["cs.AI","cs.LG","cs.SE"],"title_canon_sha256":"86bda973b208d281e2da204da74e7fa6a4ec39ed5f1eeffc34588aad8fb4cf50","abstract_canon_sha256":"0d6e3b5effa3aa5d479770b752184e0d92e79784576abe50af0895e20729fdf3"},"schema_version":"1.0"},"canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:43:25.246618Z","signature_b64":"cEsYGg4+zy8ZL6ixGsCZPQ755VBlhv9AYb2z2Q1+D+ieWxzeGVok2oGjIPpDzxZWxbbRt5VzXQTJR+j2lKVODQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","last_reissued_at":"2026-07-05T10:43:25.246129Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:43:25.246129Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.04667","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:43:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ytO5nyPGeCvRZCcwlFcYDTrUdMQEt09YPHIBbdObkBTjpIRFQ6BldEj9bsR34riVjP33N+enwSOGo/iwIuSRCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:51:15.199593Z"},"content_sha256":"7f7a1b5bd950262fc7a3e5decc860e43e66819561e3022acd4e4d675affa01dc","schema_version":"1.0","event_id":"sha256:7f7a1b5bd950262fc7a3e5decc860e43e66819561e3022acd4e4d675affa01dc"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:STT55N72ETHKS2URH5BEV45X5I","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Non-Determinism of \"Deterministic\" LLM Settings","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.SE"],"primary_cat":"cs.CL","authors_text":"Adam Sloan, Alexa Chittams, Berk Atil, Breck Baldwin, Evan Radcliffe, Ferhan Ture, Guru Rajan Rajagopal, Lisheng Fu, Lixinyu Xu, Rebecca J. Passonneau, Sarp Aykent, Tomasz Tudrej, Zhe Wu","submitted_at":"2024-08-06T16:43:35Z","abstract_excerpt":"LLM (large language model) practitioners commonly notice that outputs can vary for the same inputs under settings expected to be deterministic. Yet the questions of how pervasive this is, and with what impact on results, have not to our knowledge been systematically investigated. We investigate non-determinism in five LLMs configured to be deterministic when applied to eight common tasks in across 10 runs, in both zero-shot and few-shot settings. We see accuracy variations up to 15% across naturally occurring runs with a gap of best possible performance to worst possible performance up to 70%."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.04667","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.04667/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:43:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rxP03ujebHznRtbpEYy5XJQjkSiIvXQ2Wyc2kjFaVjL/WmIS2e4g2+GdmnckWIz0jMwB39Z9hvXWxg0B5rPxBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T19:51:15.200018Z"},"content_sha256":"c20c1a057ee7c0564de39a9f79c78ae3a078bcad1c6fbd62a51fbdca233741f5","schema_version":"1.0","event_id":"sha256:c20c1a057ee7c0564de39a9f79c78ae3a078bcad1c6fbd62a51fbdca233741f5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/bundle.json","state_url":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/STT55N72ETHKS2URH5BEV45X5I/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T19:51:15Z","links":{"resolver":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I","bundle":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/bundle.json","state":"https://pith.science/pith/STT55N72ETHKS2URH5BEV45X5I/state.json","well_known_bundle":"https://pith.science/.well-known/pith/STT55N72ETHKS2URH5BEV45X5I/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:STT55N72ETHKS2URH5BEV45X5I","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0d6e3b5effa3aa5d479770b752184e0d92e79784576abe50af0895e20729fdf3","cross_cats_sorted":["cs.AI","cs.LG","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","title_canon_sha256":"86bda973b208d281e2da204da74e7fa6a4ec39ed5f1eeffc34588aad8fb4cf50"},"schema_version":"1.0","source":{"id":"2408.04667","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.04667","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"arxiv_version","alias_value":"2408.04667v5","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.04667","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_12","alias_value":"STT55N72ETHK","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_16","alias_value":"STT55N72ETHKS2UR","created_at":"2026-07-05T10:43:25Z"},{"alias_kind":"pith_short_8","alias_value":"STT55N72","created_at":"2026-07-05T10:43:25Z"}],"graph_snapshots":[{"event_id":"sha256:c20c1a057ee7c0564de39a9f79c78ae3a078bcad1c6fbd62a51fbdca233741f5","target":"graph","created_at":"2026-07-05T10:43:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.04667/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"LLM (large language model) practitioners commonly notice that outputs can vary for the same inputs under settings expected to be deterministic. Yet the questions of how pervasive this is, and with what impact on results, have not to our knowledge been systematically investigated. We investigate non-determinism in five LLMs configured to be deterministic when applied to eight common tasks in across 10 runs, in both zero-shot and few-shot settings. We see accuracy variations up to 15% across naturally occurring runs with a gap of best possible performance to worst possible performance up to 70%.","authors_text":"Adam Sloan, Alexa Chittams, Berk Atil, Breck Baldwin, Evan Radcliffe, Ferhan Ture, Guru Rajan Rajagopal, Lisheng Fu, Lixinyu Xu, Rebecca J. Passonneau, Sarp Aykent, Tomasz Tudrej, Zhe Wu","cross_cats":["cs.AI","cs.LG","cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","title":"Non-Determinism of \"Deterministic\" LLM Settings"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.04667","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7f7a1b5bd950262fc7a3e5decc860e43e66819561e3022acd4e4d675affa01dc","target":"record","created_at":"2026-07-05T10:43:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0d6e3b5effa3aa5d479770b752184e0d92e79784576abe50af0895e20729fdf3","cross_cats_sorted":["cs.AI","cs.LG","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-06T16:43:35Z","title_canon_sha256":"86bda973b208d281e2da204da74e7fa6a4ec39ed5f1eeffc34588aad8fb4cf50"},"schema_version":"1.0","source":{"id":"2408.04667","kind":"arxiv","version":5}},"canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"94e7deb7fa24cea96a913f424af3b7ea0d60eab200cde3f741b8e01e29d99b61","first_computed_at":"2026-07-05T10:43:25.246129Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:43:25.246129Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cEsYGg4+zy8ZL6ixGsCZPQ755VBlhv9AYb2z2Q1+D+ieWxzeGVok2oGjIPpDzxZWxbbRt5VzXQTJR+j2lKVODQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:43:25.246618Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.04667","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7f7a1b5bd950262fc7a3e5decc860e43e66819561e3022acd4e4d675affa01dc","sha256:c20c1a057ee7c0564de39a9f79c78ae3a078bcad1c6fbd62a51fbdca233741f5"],"state_sha256":"8b7ac9eb91db16c5376477fbd0ab8b1c66d2ea2e17eac2fd63080cb6126ee53c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HFsQSHshT3VHmCQm5nR9fgiTPSXqNRl2ScJD+tqe6/usp04f0jDE2AF/nv9PNhEGXsxlqVk07gkrc69DgPRmAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T19:51:15.202661Z","bundle_sha256":"9b44317c4e68ae0294a20bb0d34244706fed50a57ba30cf6a65f0e996654b221"}}