{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:XWX6Y5XSBXE6N7NWYDKZMJPFYS","short_pith_number":"pith:XWX6Y5XS","canonical_record":{"source":{"id":"2503.02574","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2025-03-04T12:55:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"33eb69ba68e53d31956220eab9223c0b406280918aab3f6573c2c7c287bfcca8","abstract_canon_sha256":"79cebfcc610c45967a9f2cf54aba8fda47ac83d97cf97fa1dd2ad7f3a5366354"},"schema_version":"1.0"},"canonical_sha256":"bdafec76f20dc9e6fdb6c0d59625e5c48cbfd75914d4e810fa2bbb70fd711962","source":{"kind":"arxiv","id":"2503.02574","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.02574","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"arxiv_version","alias_value":"2503.02574v2","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.02574","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_12","alias_value":"XWX6Y5XSBXE6","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_16","alias_value":"XWX6Y5XSBXE6N7NW","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_8","alias_value":"XWX6Y5XS","created_at":"2026-05-20T00:05:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:XWX6Y5XSBXE6N7NWYDKZMJPFYS","target":"record","payload":{"canonical_record":{"source":{"id":"2503.02574","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2025-03-04T12:55:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"33eb69ba68e53d31956220eab9223c0b406280918aab3f6573c2c7c287bfcca8","abstract_canon_sha256":"79cebfcc610c45967a9f2cf54aba8fda47ac83d97cf97fa1dd2ad7f3a5366354"},"schema_version":"1.0"},"canonical_sha256":"bdafec76f20dc9e6fdb6c0d59625e5c48cbfd75914d4e810fa2bbb70fd711962","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-20T00:05:27.507600Z","signature_b64":"JjZwZs+Vi0pbi5/A6TM0sdFFwQWJ7fKTPr4b9cYn5qRdsNuGQgtU0cbpGi2Ee09yojVjhUPl1ob3may6rIrVDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bdafec76f20dc9e6fdb6c0d59625e5c48cbfd75914d4e810fa2bbb70fd711962","last_reissued_at":"2026-05-20T00:05:27.506751Z","signature_status":"signed_v1","first_computed_at":"2026-05-20T00:05:27.506751Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.02574","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-20T00:05:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"m4R/U3G1i8yn/u5FvrPOKTo7CEpLLtgy18UxKJt694jJUt8itm0TAZS2FLjML+wFqynPs+l1rjI56evLEJ26Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-10T13:53:44.101293Z"},"content_sha256":"14b46e4b33ef612cb1695b83fade31a2af4fc4556018b8fab003b52e5b8e0ae6","schema_version":"1.0","event_id":"sha256:14b46e4b33ef612cb1695b83fade31a2af4fc4556018b8fab003b52e5b8e0ae6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:XWX6Y5XSBXE6N7NWYDKZMJPFYS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLM-Safety Evaluations Lack Robustness","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CR","authors_text":"Gauthier Gidel, Leo Schwinn, Simon Geisler, Sophie Xhonneux, Stephan G\\\"unnemann, Tim Beyer","submitted_at":"2025-03-04T12:55:07Z","abstract_excerpt":"In this paper, we argue that current safety alignment research efforts for large language models are hindered by many intertwined sources of noise, such as small datasets, methodological inconsistencies, and unreliable evaluation setups. This can, at times, make it impossible to evaluate and compare attacks and defenses fairly, thereby slowing progress. We systematically analyze the LLM safety evaluation pipeline, covering dataset curation, optimization strategies for automated red-teaming, response generation, and response evaluation using LLM judges. At each stage, we identify key issues and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.02574","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.02574/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-20T00:05:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WJ5GvTv4c7DZ/muZ8mN5CdwZUTyRnjXjeEIFn29s6+naakCSo+imUGkXi8X3fZI/KozvG8kt1EFSEptA2dOUAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-10T13:53:44.102075Z"},"content_sha256":"d6cd5d8ef2b8bc0b8293f726e82b35866d8f1ec7a71ebb5f585dc245245441fa","schema_version":"1.0","event_id":"sha256:d6cd5d8ef2b8bc0b8293f726e82b35866d8f1ec7a71ebb5f585dc245245441fa"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/bundle.json","state_url":"https://pith.science/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-10T13:53:44Z","links":{"resolver":"https://pith.science/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS","bundle":"https://pith.science/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/bundle.json","state":"https://pith.science/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XWX6Y5XSBXE6N7NWYDKZMJPFYS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:XWX6Y5XSBXE6N7NWYDKZMJPFYS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"79cebfcc610c45967a9f2cf54aba8fda47ac83d97cf97fa1dd2ad7f3a5366354","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2025-03-04T12:55:07Z","title_canon_sha256":"33eb69ba68e53d31956220eab9223c0b406280918aab3f6573c2c7c287bfcca8"},"schema_version":"1.0","source":{"id":"2503.02574","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.02574","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"arxiv_version","alias_value":"2503.02574v2","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.02574","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_12","alias_value":"XWX6Y5XSBXE6","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_16","alias_value":"XWX6Y5XSBXE6N7NW","created_at":"2026-05-20T00:05:27Z"},{"alias_kind":"pith_short_8","alias_value":"XWX6Y5XS","created_at":"2026-05-20T00:05:27Z"}],"graph_snapshots":[{"event_id":"sha256:d6cd5d8ef2b8bc0b8293f726e82b35866d8f1ec7a71ebb5f585dc245245441fa","target":"graph","created_at":"2026-05-20T00:05:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.02574/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we argue that current safety alignment research efforts for large language models are hindered by many intertwined sources of noise, such as small datasets, methodological inconsistencies, and unreliable evaluation setups. This can, at times, make it impossible to evaluate and compare attacks and defenses fairly, thereby slowing progress. We systematically analyze the LLM safety evaluation pipeline, covering dataset curation, optimization strategies for automated red-teaming, response generation, and response evaluation using LLM judges. At each stage, we identify key issues and","authors_text":"Gauthier Gidel, Leo Schwinn, Simon Geisler, Sophie Xhonneux, Stephan G\\\"unnemann, Tim Beyer","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2025-03-04T12:55:07Z","title":"LLM-Safety Evaluations Lack Robustness"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.02574","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:14b46e4b33ef612cb1695b83fade31a2af4fc4556018b8fab003b52e5b8e0ae6","target":"record","created_at":"2026-05-20T00:05:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"79cebfcc610c45967a9f2cf54aba8fda47ac83d97cf97fa1dd2ad7f3a5366354","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2025-03-04T12:55:07Z","title_canon_sha256":"33eb69ba68e53d31956220eab9223c0b406280918aab3f6573c2c7c287bfcca8"},"schema_version":"1.0","source":{"id":"2503.02574","kind":"arxiv","version":2}},"canonical_sha256":"bdafec76f20dc9e6fdb6c0d59625e5c48cbfd75914d4e810fa2bbb70fd711962","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bdafec76f20dc9e6fdb6c0d59625e5c48cbfd75914d4e810fa2bbb70fd711962","first_computed_at":"2026-05-20T00:05:27.506751Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-20T00:05:27.506751Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JjZwZs+Vi0pbi5/A6TM0sdFFwQWJ7fKTPr4b9cYn5qRdsNuGQgtU0cbpGi2Ee09yojVjhUPl1ob3may6rIrVDQ==","signature_status":"signed_v1","signed_at":"2026-05-20T00:05:27.507600Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.02574","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:14b46e4b33ef612cb1695b83fade31a2af4fc4556018b8fab003b52e5b8e0ae6","sha256:d6cd5d8ef2b8bc0b8293f726e82b35866d8f1ec7a71ebb5f585dc245245441fa"],"state_sha256":"d21ea4a1cc412aeda0cba8ffbf1af13f5d42bba22f537248ad56d8cbc1eab9d6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Z0KK8ZpeC/Vlf22uVJXPyKqvq9Ym7Se6gxKntdWMoJmrPwjcOzG8kCq2t8Fc5VCPsDFPaVqPjk6UveJIWHFxAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-10T13:53:44.106885Z","bundle_sha256":"4c80ff227d6dc44341dbf26ce23b4406b307c95f21ffa678b24c5f2a04a27b26"}}