{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:NXMSC7CR4YU3XWQ4SNJ55ZW3SO","short_pith_number":"pith:NXMSC7CR","canonical_record":{"source":{"id":"2308.02828","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2023-08-05T09:30:33Z","cross_cats_sorted":[],"title_canon_sha256":"cd4f6d9415731fe4d733b313d8d255ad95618e59779f6a272ad0c9c8421139f0","abstract_canon_sha256":"c892c63bb85bab089fbeb18141ebc7080ea4f9509a608a2d28464561951d45c0"},"schema_version":"1.0"},"canonical_sha256":"6dd9217c51e629bbda1c9353dee6db93aafb78c37ce6ec420da6e4eaf3cb216c","source":{"kind":"arxiv","id":"2308.02828","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.02828","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"arxiv_version","alias_value":"2308.02828v2","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.02828","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_12","alias_value":"NXMSC7CR4YU3","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_16","alias_value":"NXMSC7CR4YU3XWQ4","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_8","alias_value":"NXMSC7CR","created_at":"2026-07-05T09:21:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:NXMSC7CR4YU3XWQ4SNJ55ZW3SO","target":"record","payload":{"canonical_record":{"source":{"id":"2308.02828","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2023-08-05T09:30:33Z","cross_cats_sorted":[],"title_canon_sha256":"cd4f6d9415731fe4d733b313d8d255ad95618e59779f6a272ad0c9c8421139f0","abstract_canon_sha256":"c892c63bb85bab089fbeb18141ebc7080ea4f9509a608a2d28464561951d45c0"},"schema_version":"1.0"},"canonical_sha256":"6dd9217c51e629bbda1c9353dee6db93aafb78c37ce6ec420da6e4eaf3cb216c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:37.897882Z","signature_b64":"gWb9b+f824zIN280748suj+Fi0r7Xg87tGRzBddvgDsT9uHdH47sX5y4EdnbRb4cHklklQ4FLIuzYQrENY+/Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6dd9217c51e629bbda1c9353dee6db93aafb78c37ce6ec420da6e4eaf3cb216c","last_reissued_at":"2026-07-05T09:21:37.897421Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:37.897421Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.02828","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AoKSnJHoHiTBIVMn++45KX/Rk3SdYVpHMuKOI6TFb3ki/EN/1yIwCSvjE6gbCwd+BeWyEKmfvX/WuSaMwt1yCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T12:10:04.238893Z"},"content_sha256":"05ca42fae4cde8da112a04b91c04bdcebb729386e13e04587614c44987339e52","schema_version":"1.0","event_id":"sha256:05ca42fae4cde8da112a04b91c04bdcebb729386e13e04587614c44987339e52"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:NXMSC7CR4YU3XWQ4SNJ55ZW3SO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An Empirical Study of the Non-determinism of ChatGPT in Code Generation","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Jie M. Zhang, Mark Harman, Meng Wang, Shuyin Ouyang","submitted_at":"2023-08-05T09:30:33Z","abstract_excerpt":"There has been a recent explosion of research on Large Language Models (LLMs) for software engineering tasks, in particular code generation. However, results from LLMs can be highly unstable; nondeterministically returning very different codes for the same prompt. Non-determinism is a potential menace to scientific conclusion validity. When non-determinism is high, scientific conclusions simply cannot be relied upon unless researchers change their behaviour to control for it in their empirical analyses. This paper conducts an empirical study to demonstrate that non-determinism is, indeed, high"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.02828","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.02828/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e3H3JA4N19d2wp7ckxJXKS3K8GKDxE/dEba5eIebJl+hKNC+I3jIAMYZHD04IMvWdMCuEcWl1kA0dNfDR784Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T12:10:04.239380Z"},"content_sha256":"8e121d7346b31eaeba2e979a222a4870b67aee3d520b0c9b4af7c8cd08e6d6bf","schema_version":"1.0","event_id":"sha256:8e121d7346b31eaeba2e979a222a4870b67aee3d520b0c9b4af7c8cd08e6d6bf"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/bundle.json","state_url":"https://pith.science/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T12:10:04Z","links":{"resolver":"https://pith.science/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO","bundle":"https://pith.science/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/bundle.json","state":"https://pith.science/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NXMSC7CR4YU3XWQ4SNJ55ZW3SO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:NXMSC7CR4YU3XWQ4SNJ55ZW3SO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c892c63bb85bab089fbeb18141ebc7080ea4f9509a608a2d28464561951d45c0","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2023-08-05T09:30:33Z","title_canon_sha256":"cd4f6d9415731fe4d733b313d8d255ad95618e59779f6a272ad0c9c8421139f0"},"schema_version":"1.0","source":{"id":"2308.02828","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.02828","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"arxiv_version","alias_value":"2308.02828v2","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.02828","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_12","alias_value":"NXMSC7CR4YU3","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_16","alias_value":"NXMSC7CR4YU3XWQ4","created_at":"2026-07-05T09:21:37Z"},{"alias_kind":"pith_short_8","alias_value":"NXMSC7CR","created_at":"2026-07-05T09:21:37Z"}],"graph_snapshots":[{"event_id":"sha256:8e121d7346b31eaeba2e979a222a4870b67aee3d520b0c9b4af7c8cd08e6d6bf","target":"graph","created_at":"2026-07-05T09:21:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.02828/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"There has been a recent explosion of research on Large Language Models (LLMs) for software engineering tasks, in particular code generation. However, results from LLMs can be highly unstable; nondeterministically returning very different codes for the same prompt. Non-determinism is a potential menace to scientific conclusion validity. When non-determinism is high, scientific conclusions simply cannot be relied upon unless researchers change their behaviour to control for it in their empirical analyses. This paper conducts an empirical study to demonstrate that non-determinism is, indeed, high","authors_text":"Jie M. Zhang, Mark Harman, Meng Wang, Shuyin Ouyang","cross_cats":[],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2023-08-05T09:30:33Z","title":"An Empirical Study of the Non-determinism of ChatGPT in Code Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.02828","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:05ca42fae4cde8da112a04b91c04bdcebb729386e13e04587614c44987339e52","target":"record","created_at":"2026-07-05T09:21:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c892c63bb85bab089fbeb18141ebc7080ea4f9509a608a2d28464561951d45c0","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.SE","submitted_at":"2023-08-05T09:30:33Z","title_canon_sha256":"cd4f6d9415731fe4d733b313d8d255ad95618e59779f6a272ad0c9c8421139f0"},"schema_version":"1.0","source":{"id":"2308.02828","kind":"arxiv","version":2}},"canonical_sha256":"6dd9217c51e629bbda1c9353dee6db93aafb78c37ce6ec420da6e4eaf3cb216c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6dd9217c51e629bbda1c9353dee6db93aafb78c37ce6ec420da6e4eaf3cb216c","first_computed_at":"2026-07-05T09:21:37.897421Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:21:37.897421Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gWb9b+f824zIN280748suj+Fi0r7Xg87tGRzBddvgDsT9uHdH47sX5y4EdnbRb4cHklklQ4FLIuzYQrENY+/Cw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:21:37.897882Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.02828","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:05ca42fae4cde8da112a04b91c04bdcebb729386e13e04587614c44987339e52","sha256:8e121d7346b31eaeba2e979a222a4870b67aee3d520b0c9b4af7c8cd08e6d6bf"],"state_sha256":"15108b5623b462be0bb2e895c821f92ef6d4d2473a8fcd1cac84095bad068b45"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hCEItZAWkQJpMIfy8B+1k8GtlwckdLEL+DUHdbu7gmMgtKUyrjVsFWleWNpWrYgQGM5zn/frYQIkzQkxwe4YCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T12:10:04.242802Z","bundle_sha256":"fe90ecb4a886deabc0c01eeeca52dce9ca29a97d53f9eba9eea0e1a5f19a301a"}}