{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:OSYBTQOAZUK6KLVHIV2OGALHM3","short_pith_number":"pith:OSYBTQOA","canonical_record":{"source":{"id":"2406.12334","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-18T06:59:24Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"859e429c5c32a70721a768b3b2ffbdf45b2dc1e4b23002fa0317901fd1df7315","abstract_canon_sha256":"b3d7b7bed1e4d9bc6d344c14b8ba311e6f88a4438d508166ee98bb0f4fbec004"},"schema_version":"1.0"},"canonical_sha256":"74b019c1c0cd15e52ea74574e3016766da0b4ece67bca066e707884f2268bbc4","source":{"kind":"arxiv","id":"2406.12334","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12334","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12334v4","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12334","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_12","alias_value":"OSYBTQOAZUK6","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_16","alias_value":"OSYBTQOAZUK6KLVH","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_8","alias_value":"OSYBTQOA","created_at":"2026-07-05T11:57:52Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:OSYBTQOAZUK6KLVHIV2OGALHM3","target":"record","payload":{"canonical_record":{"source":{"id":"2406.12334","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-18T06:59:24Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"859e429c5c32a70721a768b3b2ffbdf45b2dc1e4b23002fa0317901fd1df7315","abstract_canon_sha256":"b3d7b7bed1e4d9bc6d344c14b8ba311e6f88a4438d508166ee98bb0f4fbec004"},"schema_version":"1.0"},"canonical_sha256":"74b019c1c0cd15e52ea74574e3016766da0b4ece67bca066e707884f2268bbc4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:57:52.914142Z","signature_b64":"cXZCWym0xEGuy/igYD5AKzowvwGcUlYSEodhO4J15lYUGLz6ItT3/QbUpIoMBD2/9qSb3K1r4O24dwlPHX6QAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"74b019c1c0cd15e52ea74574e3016766da0b4ece67bca066e707884f2268bbc4","last_reissued_at":"2026-07-05T11:57:52.913655Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:57:52.913655Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.12334","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:57:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YySuZLT3fC7iNLT1OMIg0Ig5A/WHfC2ukbxAhbJGgp3vo8Evp1ZnaYvPeUsdoDCCJayMmAYGmJ7XNbrWYkBJCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T20:10:01.615673Z"},"content_sha256":"8ac81de630327383f77f54e3e3a1220b8345efeb86f1f23e1a6373a2235c2fb5","schema_version":"1.0","event_id":"sha256:8ac81de630327383f77f54e3e3a1220b8345efeb86f1f23e1a6373a2235c2fb5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:OSYBTQOAZUK6KLVHIV2OGALHM3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"What Did I Do Wrong? Quantifying LLMs' Sensitivity and Consistency to Prompt Engineering","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.SE"],"primary_cat":"cs.LG","authors_text":"Davide Sanvito, Federico Errica, Giuseppe Siracusano, Roberto Bifulco","submitted_at":"2024-06-18T06:59:24Z","abstract_excerpt":"Large Language Models (LLMs) changed the way we design and interact with software systems. Their ability to process and extract information from text has drastically improved productivity in a number of routine tasks. Developers that want to include these models in their software stack, however, face a dreadful challenge: debugging LLMs' inconsistent behavior across minor variations of the prompt. We therefore introduce two metrics for classification tasks, namely sensitivity and consistency, which are complementary to task performance. First, sensitivity measures changes of predictions across"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12334","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.12334/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:57:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"X4jVgOCGocc4Vc7zv9sMBZxY1TUKwDkq9ZJnVxkbwM64pvF3IP1PPD/H4MiEROgQVI7NFWOrdGNHufMokPUhBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T20:10:01.616630Z"},"content_sha256":"a4ee742e50c2f51b36faa7c061d24b6dd78a140440a37c48b6520ddb7aecdbbc","schema_version":"1.0","event_id":"sha256:a4ee742e50c2f51b36faa7c061d24b6dd78a140440a37c48b6520ddb7aecdbbc"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/bundle.json","state_url":"https://pith.science/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T20:10:01Z","links":{"resolver":"https://pith.science/pith/OSYBTQOAZUK6KLVHIV2OGALHM3","bundle":"https://pith.science/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/bundle.json","state":"https://pith.science/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OSYBTQOAZUK6KLVHIV2OGALHM3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:OSYBTQOAZUK6KLVHIV2OGALHM3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b3d7b7bed1e4d9bc6d344c14b8ba311e6f88a4438d508166ee98bb0f4fbec004","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-18T06:59:24Z","title_canon_sha256":"859e429c5c32a70721a768b3b2ffbdf45b2dc1e4b23002fa0317901fd1df7315"},"schema_version":"1.0","source":{"id":"2406.12334","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12334","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12334v4","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12334","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_12","alias_value":"OSYBTQOAZUK6","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_16","alias_value":"OSYBTQOAZUK6KLVH","created_at":"2026-07-05T11:57:52Z"},{"alias_kind":"pith_short_8","alias_value":"OSYBTQOA","created_at":"2026-07-05T11:57:52Z"}],"graph_snapshots":[{"event_id":"sha256:a4ee742e50c2f51b36faa7c061d24b6dd78a140440a37c48b6520ddb7aecdbbc","target":"graph","created_at":"2026-07-05T11:57:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.12334/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) changed the way we design and interact with software systems. Their ability to process and extract information from text has drastically improved productivity in a number of routine tasks. Developers that want to include these models in their software stack, however, face a dreadful challenge: debugging LLMs' inconsistent behavior across minor variations of the prompt. We therefore introduce two metrics for classification tasks, namely sensitivity and consistency, which are complementary to task performance. First, sensitivity measures changes of predictions across","authors_text":"Davide Sanvito, Federico Errica, Giuseppe Siracusano, Roberto Bifulco","cross_cats":["cs.SE"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-18T06:59:24Z","title":"What Did I Do Wrong? Quantifying LLMs' Sensitivity and Consistency to Prompt Engineering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12334","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8ac81de630327383f77f54e3e3a1220b8345efeb86f1f23e1a6373a2235c2fb5","target":"record","created_at":"2026-07-05T11:57:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b3d7b7bed1e4d9bc6d344c14b8ba311e6f88a4438d508166ee98bb0f4fbec004","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-18T06:59:24Z","title_canon_sha256":"859e429c5c32a70721a768b3b2ffbdf45b2dc1e4b23002fa0317901fd1df7315"},"schema_version":"1.0","source":{"id":"2406.12334","kind":"arxiv","version":4}},"canonical_sha256":"74b019c1c0cd15e52ea74574e3016766da0b4ece67bca066e707884f2268bbc4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"74b019c1c0cd15e52ea74574e3016766da0b4ece67bca066e707884f2268bbc4","first_computed_at":"2026-07-05T11:57:52.913655Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:57:52.913655Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cXZCWym0xEGuy/igYD5AKzowvwGcUlYSEodhO4J15lYUGLz6ItT3/QbUpIoMBD2/9qSb3K1r4O24dwlPHX6QAA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:57:52.914142Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.12334","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8ac81de630327383f77f54e3e3a1220b8345efeb86f1f23e1a6373a2235c2fb5","sha256:a4ee742e50c2f51b36faa7c061d24b6dd78a140440a37c48b6520ddb7aecdbbc"],"state_sha256":"d7b9d0c58e9e525b453ee482b6e5db5f2405f5a71492676b2d0a5985a0ea2b7e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KYmGDQ4IEf9/GxflVzCgj9+2FPn5r0Nfy2cMTiOfnbgOANd5cXFPzdi2KytnbW0GPIa7i3VQJ4J9zc5i/079DQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T20:10:01.624075Z","bundle_sha256":"10d4a80be4165a4f729920a1d10bdf90a5b162d3983d6c59c4c5dd55190c7a84"}}